From 8238f1440c1f429bac24f0b43ade7a4a2ef07965 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Wed, 27 Apr 2016 14:30:06 +0200
Subject: [PATCH] [scripts] Add scripts to process user job splits and run
worker tasks
---
beat/web/scripts/process.py | 108 +++++++++++++++++++++++++++
beat/web/scripts/worker.py | 144 ++++++++++++++++++++++++++++++++++++
setup.py | 2 +
3 files changed, 254 insertions(+)
create mode 100644 beat/web/scripts/process.py
create mode 100644 beat/web/scripts/worker.py
diff --git a/beat/web/scripts/process.py b/beat/web/scripts/process.py
new file mode 100644
index 000000000..0826b11e6
--- /dev/null
+++ b/beat/web/scripts/process.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+"""\
+Processes one split.
+
+Usage:
+ %(prog)s [--settings=<file>] [--cpulimit=<file>] [-v ...] <execute> <split>
+ %(prog)s (-h | --help)
+ %(prog)s (-V | --version)
+
+Arguments:
+
+
+ <execute> The path to the base execution program for running the user code
+ <split> The primary-key of the split to treat by this subprocess
+
+
+Options:
+ -h, --help Show this help message
+ -V, --version Show program's version number
+ -v, --verbose Increases the output verbosity level
+ -S <file>, --settings=<file> The module name to the Django settings file
+ [default: beat.web.settings.settings]
+ -C <file>, --cpulimit=<file> The path to the cpulimit program to use. If
+ not set, CPU limiting is not enforced.
+
+
+Examples:
+
+ To start the job split processing do the following:
+
+ $ %(prog)s <path-to-execute> <split-id>
+
+ You can optionally pass the ``-v`` flag to start the worker with the logging
+ level set to ``INFO`` or ``-vv`` to set it to ``DEBUG``. By default, the
+ logging level is set to ``WARNING`` if no ``-v`` flag is passed.
+
+ You can optionally also set the path to the ``cpulimit`` program to use. If
+ it is not set, then CPU limiting will not be enforced.
+
+"""
+
+
+import logging
+__logging_format__='[%(levelname)s] %(message)s'
+logging.basicConfig(format=__logging_format__)
+logger = logging.getLogger(__name__)
+
+
+import os
+import sys
+import docopt
+
+
+def main(user_input=None):
+
+ arguments = docopt.docopt(
+ __doc__ % dict(
+ prog=os.path.basename(sys.argv[0]),
+ ),
+ )
+
+
+ # Sets-up logging
+ if arguments['--verbose'] == 1:
+ logging.getLogger().setLevel(logging.INFO)
+ elif arguments['--verbose'] >= 2:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ # Initializes the Django framework
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', arguments['--settings'])
+ from django.conf import settings
+ from django import setup
+ setup()
+
+ from beat.web.backend.schedule import process
+
+ process(
+ split_pk=int(arguments['<split>']),
+ execute=arguments['<execute>'],
+ cpulimit=arguments['--cpulimit'],
+ )
diff --git a/beat/web/scripts/worker.py b/beat/web/scripts/worker.py
new file mode 100644
index 000000000..21f447298
--- /dev/null
+++ b/beat/web/scripts/worker.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+"""\
+Starts the worker process.
+
+Usage:
+ %(prog)s [-v ... | --verbose ...] [--settings=<file>]
+ %(prog)s [--cpulimit=<file>] [--environments=<path>]
+ %(prog)s (-h | --help)
+ %(prog)s (-V | --version)
+
+
+Options:
+ -h, --help Show this help message
+ -V, --version Show program's version number
+ -v, --verbose Increases the output verbosity level
+ -S <file>, --settings=<file> The module name to the Django settings
+ file [default: beat.web.settings.settings]
+ -c <file>, --cpulimit=<file> The path to the cpulimit program to use. If
+ not set, try to search in standard
+ locations. If not found, CPU limiting is
+ not enforced.
+ -e <path>, --environments=<path> The path to the installation root of
+ available environments.
+
+
+Examples:
+
+ To start the worker do the following:
+
+ $ %(prog)s
+
+ You can pass the ``-v`` flag to start the worker with the logging level set
+ to ``INFO`` or ``-vv`` to set it to ``DEBUG``. By default, the logging level
+ is set to ``WARNING`` if no ``-v`` flag is passed.
+
+"""
+
+import logging
+__logging_format__='[%(levelname)s] %(message)s'
+logging.basicConfig(format=__logging_format__)
+logger = logging.getLogger(__name__)
+
+
+import os
+import sys
+import time
+import signal
+import docopt
+
+
+def main(user_input=None):
+
+ stop = False
+
+ # installs SIGTERM handler
+ def handler(signum, frame):
+ stop = True
+ signal.signal(signal.SIGTERM, handler)
+
+ arguments = docopt.docopt(
+ __doc__ % dict(
+ prog=os.path.basename(sys.argv[0]),
+ ),
+ )
+
+
+ # Sets-up logging
+ if arguments['--verbose'] == 1:
+ logging.getLogger().setLevel(logging.INFO)
+ elif arguments['--verbose'] >= 2:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+
+ try:
+
+ from beat.core.async import resolve_cpulimit_path
+ cpulimit = resolve_cpulimit_path(arguments['--cpulimit'])
+
+ from beat.web.backend.schedule import find_environments, work
+ from beat.web.backend.schedule import resolve_process_path
+ from beat.web.backend.schedule import worker_shutdown, worker_update
+
+ process = resolve_process_path()
+ environments = find_environments(arguments['--environments'] or [])
+
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', arguments['--settings'])
+ from django.conf import settings
+
+ from django import setup
+ setup()
+
+ worker_update()
+ last_update = time.time()
+
+ timing = settings.WORKER_INTERVAL
+
+ while not stop:
+
+ start = time.time()
+ work(environments, cpulimit, process)
+ currtime = time.time()
+ duration = currtime - start
+
+ if duration < settings.WORKER_INTERVAL:
+ time.sleep(settings.WORKER_INTERVAL - duration)
+
+ if (last_update - currtime) > settings.WORKER_STATE_UPDATE:
+ worker_update()
+
+ except Exception:
+ from traceback import format_exc
+ logger.error(format_exc())
+
+
+ finally:
+
+ worker_shutdown()
diff --git a/setup.py b/setup.py
index 1b7ac5060..ec6d688c0 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,8 @@ setup(
entry_points={
'console_scripts': [
'localhost.py = beat.web.scripts.localhost:main',
+ 'process.py = beat.web.scripts.process:main',
+ 'worker.py = beat.web.scripts.worker:main',
],
},
--
GitLab