[backend] Work-around gevent limitation for Django test server

beat/web/backend/tests.py

@@ -2042,9 +2042,9 @@ class Working(BaseBackendTestCase):
         from beat.core.async import resolve_cpulimit_path
         self.cpulimit = resolve_cpulimit_path(None)
 
-        from . import utils
-        self.process = utils.resolve_process_path()
-        self.environments = utils.find_environments(None)
+        from ..utils import path
+        self.process = path.resolve_process_path()
+        self.environments = path.find_environments(None)
         self.env1_execute = self.environments['environment (1)']['execute']
 
         if not os.path.exists(settings.CACHE_ROOT):
@@ -2497,9 +2497,9 @@ class WorkingExternally(TransactionTestCase):
         from beat.core.async import resolve_cpulimit_path
         self.cpulimit = resolve_cpulimit_path(None)
 
-        from . import utils
-        self.process = utils.resolve_process_path()
-        self.environments = utils.find_environments(None)
+        from ..utils import path
+        self.process = path.resolve_process_path()
+        self.environments = path.find_environments(None)
 
         if not os.path.exists(settings.CACHE_ROOT):
             os.makedirs(settings.CACHE_ROOT)

beat/web/backend/utils.py

@@ -352,61 +352,6 @@ def cleanup_zombies():
             child.wait()
 
 
-def resolve_process_path():
-  '''Returns the path to cpulimit'''
-
-  basedir = os.path.dirname(os.path.realpath(sys.argv[0]))
-  r = os.path.join(basedir, 'process')
-
-  if not os.path.exists(r):
-      raise RuntimeError("Cannot find `process.py' at `%s' - please check " \
-          "your installation" % basedir)
-
-  return r
-
-
-def find_environments(paths=None):
-  '''Finds list of known environments
-
-  Parameters:
-
-    paths (list, Optional): A list of paths where to search for environments.
-      If not set, then load default environments if possible.
-
-
-  Returns:
-
-    dict: A dictionary containing each environment available using as key the
-      natural key for environments (i.e., ``name (version)``) and as values
-      another dictionary with these keys:
-
-        * name: The environment name (str)
-        * version: The environment version (str)
-        * os: The output of ``uname -a`` (list):
-          1. Operating system (str)
-          2. Hostname (str)
-          3. Kernel version (str)
-          4. Kernel compilation details (str)
-          5. Platform (``x86_64`` for 64-bits or ``i386`` for 32-bits) (str)
-        * execute: The path to the ``execute`` script to be used for running
-          user jobs (str)
-        * directory: The path leading to the root of this environment (str)
-
-  '''
-
-  from beat.core.execution import discover_environments
-
-  if paths is not None:
-    logger.debug("Search for environments at `%s'", os.pathsep.join(paths))
-    return discover_environments(paths)
-
-  else:
-    import pkg_resources
-    path = pkg_resources.resource_filename(__name__, 'environments')
-    logger.debug("Search for environments at `%s'", path)
-    return discover_environments([path])
-
-
 def pick_execute(split, environments):
     """Resolves the path to the ``execute`` program to use for the split"""
 

beat/web/backend/views.py

@@ -42,8 +42,6 @@ from django.contrib.auth.decorators import login_required
 from django.http import HttpResponseForbidden
 from django.contrib import messages
 
-from beat.core.async import resolve_cpulimit_path
-
 from ..experiments.models import Experiment
 
 from .models import Environment, Worker, Queue
@@ -58,18 +56,14 @@ from . import schedule
 class Work:
     '''Helper to do the required worker job for local scheduling'''
 
-    cpulimit = None
-    process = None
-    environments = None
     worker = None
 
     def __setup__(self):
 
-        Worker.cpulimit = resolve_cpulimit_path(None)
-        Worker.process = utils.resolve_process_path()
-        Worker.environments = utils.find_environments(None)
         Work.worker = Worker.objects.get(name=socket.gethostname()) \
             if Worker.objects.count() != 1 else Worker.objects.get()
+        Work.worker.activate()
+        Work.worker.save()
 
     def __call__(self):
 
@@ -77,7 +71,8 @@ class Work:
 
         # Regular work
         utils.cleanup_zombies()
-        Work.worker.work(Work.environments, Work.cpulimit, Work.process)
+        Work.worker.work(settings.WORKER_ENVIRONMENTS,
+            settings.WORKER_CPULIMIT_PATH, settings.WORKER_PROCESS_PATH)
 
 
 #------------------------------------------------

beat/web/scripts/worker.py

@@ -112,13 +112,14 @@ def main(user_input=None):
     signal.signal(signal.SIGTERM, handler)
     signal.signal(signal.SIGINT, handler)
 
+    from ..utils import path
     from ..backend import utils
     from ..backend.models import Worker
 
     from beat.core.async import resolve_cpulimit_path
     cpulimit = resolve_cpulimit_path(arguments['--cpulimit'])
-    process = utils.resolve_process_path()
-    environments = utils.find_environments(arguments['--environments'])
+    process = path.resolve_process_path()
+    environments = path.find_environments(arguments['--environments'])
 
     timing = int(arguments['--period']) \
         if arguments['--period'] else settings.WORKER_INTERVAL

beat/web/settings/settings.py

@@ -251,6 +251,16 @@ WORKER_INTERVAL = 5 #seconds
 # manually.
 SCHEDULING_PANEL = True
 
+# If the scheduling panel must be used, we should setup a few things as well
+if SCHEDULING_PANEL:
+
+    from ..utils import path
+    from beat.core.async import resolve_cpulimit_path
+    WORKER_CPULIMIT_PATH = resolve_cpulimit_path(None)
+    WORKER_PROCESS_PATH  = path.resolve_process_path()
+    WORKER_ENVIRONMENTS  = path.find_environments(None)
+
+
 # The maximum index split errors control the maximum number of times we can
 # incur in an index split error condition without cancelling the block
 # execution altogether. This number, multiplied by the scheduling interval,

beat/web/utils/path.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+'''Utilities for path management'''
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+
+from beat.core.execution import discover_environments
+
+
+def resolve_process_path():
+  '''Returns the path to cpulimit'''
+
+  basedir = os.path.dirname(os.path.realpath(sys.argv[0]))
+  r = os.path.join(basedir, 'process')
+
+  if not os.path.exists(r):
+      raise RuntimeError("Cannot find `process' at `%s' - please check " \
+          "your installation" % basedir)
+
+  return r
+
+
+def find_environments(paths=None):
+  '''Finds list of known environments
+
+  Parameters:
+
+    paths (list, Optional): A list of paths where to search for environments.
+      If not set, then load default environments if possible.
+
+
+  Returns:
+
+    dict: A dictionary containing each environment available using as key the
+      natural key for environments (i.e., ``name (version)``) and as values
+      another dictionary with these keys:
+
+        * name: The environment name (str)
+        * version: The environment version (str)
+        * os: The output of ``uname -a`` (list):
+          1. Operating system (str)
+          2. Hostname (str)
+          3. Kernel version (str)
+          4. Kernel compilation details (str)
+          5. Platform (``x86_64`` for 64-bits or ``i386`` for 32-bits) (str)
+        * execute: The path to the ``execute`` script to be used for running
+          user jobs (str)
+        * directory: The path leading to the root of this environment (str)
+
+  '''
+
+  if paths is not None:
+    logger.debug("Search for environments at `%s'", os.pathsep.join(paths))
+    return discover_environments(paths)
+
+  else:
+    path = pkg_resources.resource_filename(__name__, 'environments')
+    logger.debug("Search for environments at `%s'", path)
+    return discover_environments([path])