diff --git a/beat/web/scripts/process.py b/beat/web/scripts/process.py new file mode 100644 index 0000000000000000000000000000000000000000..0826b11e6d1dc8cc9ab70568a6d7d6b76a9d821b --- /dev/null +++ b/beat/web/scripts/process.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : + +############################################################################### +# # +# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ # +# Contact: beat.support@idiap.ch # +# # +# This file is part of the beat.web module of the BEAT platform. # +# # +# Commercial License Usage # +# Licensees holding valid commercial BEAT licenses may use this file in # +# accordance with the terms contained in a written agreement between you # +# and Idiap. For further information contact tto@idiap.ch # +# # +# Alternatively, this file may be used under the terms of the GNU Affero # +# Public License version 3 as published by the Free Software and appearing # +# in the file LICENSE.AGPL included in the packaging of this file. # +# The BEAT platform is distributed in the hope that it will be useful, but # +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # +# or FITNESS FOR A PARTICULAR PURPOSE. # +# # +# You should have received a copy of the GNU Affero Public License along # +# with the BEAT platform. If not, see http://www.gnu.org/licenses/. # +# # +############################################################################### + + +"""\ +Processes one split. + +Usage: + %(prog)s [--settings=<file>] [--cpulimit=<file>] [-v ...] <execute> <split> + %(prog)s (-h | --help) + %(prog)s (-V | --version) + +Arguments: + + + <execute> The path to the base execution program for running the user code + <split> The primary-key of the split to treat by this subprocess + + +Options: + -h, --help Show this help message + -V, --version Show program's version number + -v, --verbose Increases the output verbosity level + -S <file>, --settings=<file> The module name to the Django settings file + [default: beat.web.settings.settings] + -C <file>, --cpulimit=<file> The path to the cpulimit program to use. If + not set, CPU limiting is not enforced. + + +Examples: + + To start the job split processing do the following: + + $ %(prog)s <path-to-execute> <split-id> + + You can optionally pass the ``-v`` flag to start the worker with the logging + level set to ``INFO`` or ``-vv`` to set it to ``DEBUG``. By default, the + logging level is set to ``WARNING`` if no ``-v`` flag is passed. + + You can optionally also set the path to the ``cpulimit`` program to use. If + it is not set, then CPU limiting will not be enforced. + +""" + + +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger(__name__) + + +import os +import sys +import docopt + + +def main(user_input=None): + + arguments = docopt.docopt( + __doc__ % dict( + prog=os.path.basename(sys.argv[0]), + ), + ) + + + # Sets-up logging + if arguments['--verbose'] == 1: + logging.getLogger().setLevel(logging.INFO) + elif arguments['--verbose'] >= 2: + logging.getLogger().setLevel(logging.DEBUG) + + # Initializes the Django framework + os.environ.setdefault('DJANGO_SETTINGS_MODULE', arguments['--settings']) + from django.conf import settings + from django import setup + setup() + + from beat.web.backend.schedule import process + + process( + split_pk=int(arguments['<split>']), + execute=arguments['<execute>'], + cpulimit=arguments['--cpulimit'], + ) diff --git a/beat/web/scripts/worker.py b/beat/web/scripts/worker.py new file mode 100644 index 0000000000000000000000000000000000000000..21f4472989841d09218b192c98e2b9369a18d37f --- /dev/null +++ b/beat/web/scripts/worker.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : + +############################################################################### +# # +# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ # +# Contact: beat.support@idiap.ch # +# # +# This file is part of the beat.web module of the BEAT platform. # +# # +# Commercial License Usage # +# Licensees holding valid commercial BEAT licenses may use this file in # +# accordance with the terms contained in a written agreement between you # +# and Idiap. For further information contact tto@idiap.ch # +# # +# Alternatively, this file may be used under the terms of the GNU Affero # +# Public License version 3 as published by the Free Software and appearing # +# in the file LICENSE.AGPL included in the packaging of this file. # +# The BEAT platform is distributed in the hope that it will be useful, but # +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # +# or FITNESS FOR A PARTICULAR PURPOSE. # +# # +# You should have received a copy of the GNU Affero Public License along # +# with the BEAT platform. If not, see http://www.gnu.org/licenses/. # +# # +############################################################################### + + +"""\ +Starts the worker process. + +Usage: + %(prog)s [-v ... | --verbose ...] [--settings=<file>] + %(prog)s [--cpulimit=<file>] [--environments=<path>] + %(prog)s (-h | --help) + %(prog)s (-V | --version) + + +Options: + -h, --help Show this help message + -V, --version Show program's version number + -v, --verbose Increases the output verbosity level + -S <file>, --settings=<file> The module name to the Django settings + file [default: beat.web.settings.settings] + -c <file>, --cpulimit=<file> The path to the cpulimit program to use. If + not set, try to search in standard + locations. If not found, CPU limiting is + not enforced. + -e <path>, --environments=<path> The path to the installation root of + available environments. + + +Examples: + + To start the worker do the following: + + $ %(prog)s + + You can pass the ``-v`` flag to start the worker with the logging level set + to ``INFO`` or ``-vv`` to set it to ``DEBUG``. By default, the logging level + is set to ``WARNING`` if no ``-v`` flag is passed. + +""" + +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger(__name__) + + +import os +import sys +import time +import signal +import docopt + + +def main(user_input=None): + + stop = False + + # installs SIGTERM handler + def handler(signum, frame): + stop = True + signal.signal(signal.SIGTERM, handler) + + arguments = docopt.docopt( + __doc__ % dict( + prog=os.path.basename(sys.argv[0]), + ), + ) + + + # Sets-up logging + if arguments['--verbose'] == 1: + logging.getLogger().setLevel(logging.INFO) + elif arguments['--verbose'] >= 2: + logging.getLogger().setLevel(logging.DEBUG) + + + try: + + from beat.core.async import resolve_cpulimit_path + cpulimit = resolve_cpulimit_path(arguments['--cpulimit']) + + from beat.web.backend.schedule import find_environments, work + from beat.web.backend.schedule import resolve_process_path + from beat.web.backend.schedule import worker_shutdown, worker_update + + process = resolve_process_path() + environments = find_environments(arguments['--environments'] or []) + + os.environ.setdefault('DJANGO_SETTINGS_MODULE', arguments['--settings']) + from django.conf import settings + + from django import setup + setup() + + worker_update() + last_update = time.time() + + timing = settings.WORKER_INTERVAL + + while not stop: + + start = time.time() + work(environments, cpulimit, process) + currtime = time.time() + duration = currtime - start + + if duration < settings.WORKER_INTERVAL: + time.sleep(settings.WORKER_INTERVAL - duration) + + if (last_update - currtime) > settings.WORKER_STATE_UPDATE: + worker_update() + + except Exception: + from traceback import format_exc + logger.error(format_exc()) + + + finally: + + worker_shutdown() diff --git a/setup.py b/setup.py index 1b7ac50604efe960ab6412295ab38a0ed8ebc2e6..ec6d688c0bbf8833305c09139c78c787885989f5 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,8 @@ setup( entry_points={ 'console_scripts': [ 'localhost.py = beat.web.scripts.localhost:main', + 'process.py = beat.web.scripts.process:main', + 'worker.py = beat.web.scripts.worker:main', ], },