From 34569066a9ca97a186839944390a05cf63c7b208 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Sun, 8 May 2016 21:53:27 +0200
Subject: [PATCH] [backend] Work-around gevent limitation for Django test
server
---
beat/web/backend/tests.py | 12 +--
beat/web/backend/utils.py | 55 ------------
beat/web/backend/views.py | 13 +--
beat/web/scripts/worker.py | 5 +-
beat/web/settings/settings.py | 10 +++
.../{backend => utils}/environments/default | 0
beat/web/utils/path.py | 89 +++++++++++++++++++
7 files changed, 112 insertions(+), 72 deletions(-)
rename beat/web/{backend => utils}/environments/default (100%)
create mode 100644 beat/web/utils/path.py
diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py
index faa4f8424..792619117 100644
--- a/beat/web/backend/tests.py
+++ b/beat/web/backend/tests.py
@@ -2042,9 +2042,9 @@ class Working(BaseBackendTestCase):
from beat.core.async import resolve_cpulimit_path
self.cpulimit = resolve_cpulimit_path(None)
- from . import utils
- self.process = utils.resolve_process_path()
- self.environments = utils.find_environments(None)
+ from ..utils import path
+ self.process = path.resolve_process_path()
+ self.environments = path.find_environments(None)
self.env1_execute = self.environments['environment (1)']['execute']
if not os.path.exists(settings.CACHE_ROOT):
@@ -2497,9 +2497,9 @@ class WorkingExternally(TransactionTestCase):
from beat.core.async import resolve_cpulimit_path
self.cpulimit = resolve_cpulimit_path(None)
- from . import utils
- self.process = utils.resolve_process_path()
- self.environments = utils.find_environments(None)
+ from ..utils import path
+ self.process = path.resolve_process_path()
+ self.environments = path.find_environments(None)
if not os.path.exists(settings.CACHE_ROOT):
os.makedirs(settings.CACHE_ROOT)
diff --git a/beat/web/backend/utils.py b/beat/web/backend/utils.py
index 1ec4deb74..351026f8f 100644
--- a/beat/web/backend/utils.py
+++ b/beat/web/backend/utils.py
@@ -352,61 +352,6 @@ def cleanup_zombies():
child.wait()
-def resolve_process_path():
- '''Returns the path to cpulimit'''
-
- basedir = os.path.dirname(os.path.realpath(sys.argv[0]))
- r = os.path.join(basedir, 'process')
-
- if not os.path.exists(r):
- raise RuntimeError("Cannot find `process.py' at `%s' - please check " \
- "your installation" % basedir)
-
- return r
-
-
-def find_environments(paths=None):
- '''Finds list of known environments
-
- Parameters:
-
- paths (list, Optional): A list of paths where to search for environments.
- If not set, then load default environments if possible.
-
-
- Returns:
-
- dict: A dictionary containing each environment available using as key the
- natural key for environments (i.e., ``name (version)``) and as values
- another dictionary with these keys:
-
- * name: The environment name (str)
- * version: The environment version (str)
- * os: The output of ``uname -a`` (list):
- 1. Operating system (str)
- 2. Hostname (str)
- 3. Kernel version (str)
- 4. Kernel compilation details (str)
- 5. Platform (``x86_64`` for 64-bits or ``i386`` for 32-bits) (str)
- * execute: The path to the ``execute`` script to be used for running
- user jobs (str)
- * directory: The path leading to the root of this environment (str)
-
- '''
-
- from beat.core.execution import discover_environments
-
- if paths is not None:
- logger.debug("Search for environments at `%s'", os.pathsep.join(paths))
- return discover_environments(paths)
-
- else:
- import pkg_resources
- path = pkg_resources.resource_filename(__name__, 'environments')
- logger.debug("Search for environments at `%s'", path)
- return discover_environments([path])
-
-
def pick_execute(split, environments):
"""Resolves the path to the ``execute`` program to use for the split"""
diff --git a/beat/web/backend/views.py b/beat/web/backend/views.py
index 3e5780be2..6d405cad1 100644
--- a/beat/web/backend/views.py
+++ b/beat/web/backend/views.py
@@ -42,8 +42,6 @@ from django.contrib.auth.decorators import login_required
from django.http import HttpResponseForbidden
from django.contrib import messages
-from beat.core.async import resolve_cpulimit_path
-
from ..experiments.models import Experiment
from .models import Environment, Worker, Queue
@@ -58,18 +56,14 @@ from . import schedule
class Work:
'''Helper to do the required worker job for local scheduling'''
- cpulimit = None
- process = None
- environments = None
worker = None
def __setup__(self):
- Worker.cpulimit = resolve_cpulimit_path(None)
- Worker.process = utils.resolve_process_path()
- Worker.environments = utils.find_environments(None)
Work.worker = Worker.objects.get(name=socket.gethostname()) \
if Worker.objects.count() != 1 else Worker.objects.get()
+ Work.worker.activate()
+ Work.worker.save()
def __call__(self):
@@ -77,7 +71,8 @@ class Work:
# Regular work
utils.cleanup_zombies()
- Work.worker.work(Work.environments, Work.cpulimit, Work.process)
+ Work.worker.work(settings.WORKER_ENVIRONMENTS,
+ settings.WORKER_CPULIMIT_PATH, settings.WORKER_PROCESS_PATH)
#------------------------------------------------
diff --git a/beat/web/scripts/worker.py b/beat/web/scripts/worker.py
index dadc88c2d..aca6f39e9 100644
--- a/beat/web/scripts/worker.py
+++ b/beat/web/scripts/worker.py
@@ -112,13 +112,14 @@ def main(user_input=None):
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
+ from ..utils import path
from ..backend import utils
from ..backend.models import Worker
from beat.core.async import resolve_cpulimit_path
cpulimit = resolve_cpulimit_path(arguments['--cpulimit'])
- process = utils.resolve_process_path()
- environments = utils.find_environments(arguments['--environments'])
+ process = path.resolve_process_path()
+ environments = path.find_environments(arguments['--environments'])
timing = int(arguments['--period']) \
if arguments['--period'] else settings.WORKER_INTERVAL
diff --git a/beat/web/settings/settings.py b/beat/web/settings/settings.py
index 357a8efb8..ce05e9550 100644
--- a/beat/web/settings/settings.py
+++ b/beat/web/settings/settings.py
@@ -251,6 +251,16 @@ WORKER_INTERVAL = 5 #seconds
# manually.
SCHEDULING_PANEL = True
+# If the scheduling panel must be used, we should setup a few things as well
+if SCHEDULING_PANEL:
+
+ from ..utils import path
+ from beat.core.async import resolve_cpulimit_path
+ WORKER_CPULIMIT_PATH = resolve_cpulimit_path(None)
+ WORKER_PROCESS_PATH = path.resolve_process_path()
+ WORKER_ENVIRONMENTS = path.find_environments(None)
+
+
# The maximum index split errors control the maximum number of times we can
# incur in an index split error condition without cancelling the block
# execution altogether. This number, multiplied by the scheduling interval,
diff --git a/beat/web/backend/environments/default b/beat/web/utils/environments/default
similarity index 100%
rename from beat/web/backend/environments/default
rename to beat/web/utils/environments/default
diff --git a/beat/web/utils/path.py b/beat/web/utils/path.py
new file mode 100644
index 000000000..a95ba5c28
--- /dev/null
+++ b/beat/web/utils/path.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+'''Utilities for path management'''
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+
+from beat.core.execution import discover_environments
+
+
+def resolve_process_path():
+ '''Returns the path to cpulimit'''
+
+ basedir = os.path.dirname(os.path.realpath(sys.argv[0]))
+ r = os.path.join(basedir, 'process')
+
+ if not os.path.exists(r):
+ raise RuntimeError("Cannot find `process' at `%s' - please check " \
+ "your installation" % basedir)
+
+ return r
+
+
+def find_environments(paths=None):
+ '''Finds list of known environments
+
+ Parameters:
+
+ paths (list, Optional): A list of paths where to search for environments.
+ If not set, then load default environments if possible.
+
+
+ Returns:
+
+ dict: A dictionary containing each environment available using as key the
+ natural key for environments (i.e., ``name (version)``) and as values
+ another dictionary with these keys:
+
+ * name: The environment name (str)
+ * version: The environment version (str)
+ * os: The output of ``uname -a`` (list):
+ 1. Operating system (str)
+ 2. Hostname (str)
+ 3. Kernel version (str)
+ 4. Kernel compilation details (str)
+ 5. Platform (``x86_64`` for 64-bits or ``i386`` for 32-bits) (str)
+ * execute: The path to the ``execute`` script to be used for running
+ user jobs (str)
+ * directory: The path leading to the root of this environment (str)
+
+ '''
+
+ if paths is not None:
+ logger.debug("Search for environments at `%s'", os.pathsep.join(paths))
+ return discover_environments(paths)
+
+ else:
+ path = pkg_resources.resource_filename(__name__, 'environments')
+ logger.debug("Search for environments at `%s'", path)
+ return discover_environments([path])
--
GitLab