Commit 9f92f26e authored by André Anjos's avatar André Anjos 💬

Merge branch 'zmq_refactoring' into 'master'

ZMQ refactoring

Closes #518 and #517

See merge request !275
parents 7165564b 18cca41c
Pipeline #27969 passed with stages
in 16 minutes and 26 seconds
[flake8]
max-line-length = 80
select = B,C,E,F,W,T4,B9,B950
ignore = E501, W503
......@@ -15,6 +15,7 @@ sphinx/
.mr.developer.cfg
.coverage
*.sql3
*.sqlite3
.DS_Store
beat/web/settings/settings.py
src/
......
......@@ -27,7 +27,7 @@ build_linux_36:
- buildout
- python -c "from beat.core.test.utils import pull_docker_test_images as f; f()"
- export COVERAGE_FILE=.coverage.django
- ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
- ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2 --noinput
- export BEAT_CMDLINE_TEST_PLATFORM="django://beat.web.settings.ci"
- export COVERAGE_FILE=.coverage.cmdline
- export NOSE_WITH_COVERAGE=1
......
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/ambv/black
rev: stable
hooks:
- id: black
language_version: python3.6
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
- id: check-added-large-files
- id: check-docstring-first
- id: flake8
- id: check-yaml
exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/bandit
rev: 'master' # Update me!
hooks:
- id: bandit
exclude: beat/editor/test
......@@ -27,6 +27,7 @@
# Django settings for tests on the CI server
from .test import *
from .test import * # noqa
RUNNING_ON_CI = True
DATABASES["default"]["OPTIONS"]["timeout"] = 60 # noqa
......@@ -26,48 +26,64 @@
###############################################################################
# Django settings for tests
import os
import platform
import sys
from .settings import *
from .settings import * # noqa
TEST_CONFIGURATION = True
RUNNING_ON_CI = False
DEBUG = False
TEMPLATES[0]['OPTIONS']['debug'] = DEBUG
TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG # noqa
ALLOWED_HOSTS = [
'testserver',
]
ALLOWED_HOSTS = ["testserver"]
DATABASES['default']['NAME'] = 'test.sql3'
DATABASES['default']['TEST'] = {'NAME': DATABASES['default']['NAME']}
DATABASES['default']['OPTIONS']['timeout'] = 30
if platform.system() == "Linux":
shm_path = "/dev/shm/beatweb" # nosec
if not os.path.exists(shm_path):
os.makedirs(shm_path)
import sys
if 'beat.cmdline' in sys.argv:
database_name = os.path.join(shm_path, "test.sqlite3")
else:
database_name = "test.sqlite3"
DATABASES["default"]["NAME"] = database_name # noqa
DATABASES["default"]["TEST"] = {"NAME": DATABASES["default"]["NAME"]} # noqa
DATABASES["default"]["OPTIONS"]["timeout"] = 30 # noqa
DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa
# Timeout used in test when waiting for an update
DB_REFRESH_TIMEOUT = int(os.environ.get("DB_REFRESH_TIMEOUT", 10))
if "beat.cmdline" in sys.argv:
# make it in-memory for cmdline app tests
DATABASES['default']['NAME'] = ':memory:'
DATABASES["default"]["NAME"] = ":memory:" # noqa
LOGGING['handlers']['console']['level'] = 'DEBUG'
LOGGING['loggers']['beat.core']['handlers'] = ['discard']
LOGGING['loggers']['beat.web']['handlers'] = ['discard']
LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
LOGGING["handlers"]["console"]["level"] = "DEBUG" # noqa
LOGGING["loggers"]["beat.core"]["handlers"] = ["discard"] # noqa
LOGGING["loggers"]["beat.web"]["handlers"] = ["discard"] # noqa
LOGGING["loggers"]["beat.web.utils.management.commands"]["handlers"] = [ # noqa
"discard"
]
BASE_DIR = os.path.dirname(os.path.abspath(__name__))
PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
ALGORITHMS_ROOT = os.path.join(PREFIX, 'algorithms')
PLOTTERS_ROOT = os.path.join(PREFIX, 'plotters')
LIBRARIES_ROOT = os.path.join(PREFIX, 'libraries')
DATABASES_ROOT = os.path.join(PREFIX, 'databases')
DATAFORMATS_ROOT = os.path.join(PREFIX, 'dataformats')
TOOLCHAINS_ROOT = os.path.join(PREFIX, 'toolchains')
EXPERIMENTS_ROOT = os.path.join(PREFIX, 'experiments')
CACHE_ROOT = os.path.join(PREFIX, 'cache')
if platform.system() == "Linux":
default_prefix = os.path.join(shm_path, "test_prefix") # nosec
else:
default_prefix = os.path.realpath("./test_prefix")
PREFIX = os.environ.get("BEAT_TEST_PREFIX", default_prefix)
ALGORITHMS_ROOT = os.path.join(PREFIX, "algorithms")
PLOTTERS_ROOT = os.path.join(PREFIX, "plotters")
LIBRARIES_ROOT = os.path.join(PREFIX, "libraries")
DATABASES_ROOT = os.path.join(PREFIX, "databases")
DATAFORMATS_ROOT = os.path.join(PREFIX, "dataformats")
TOOLCHAINS_ROOT = os.path.join(PREFIX, "toolchains")
EXPERIMENTS_ROOT = os.path.join(PREFIX, "experiments")
CACHE_ROOT = os.path.join(PREFIX, "cache")
LOCAL_SCHEDULER_VERBOSITY = None
LOCAL_SCHEDULER_USE_DOCKER = False
# To speed-up tests, don't put this in production
PASSWORD_HASHERS = [
'django.contrib.auth.hashers.MD5PasswordHasher',
]
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# encoding: utf-8
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import logging
from django.core.management.base import BaseCommand
from django.db import transaction
from beat.web.backend.models import Worker
from beat.core.bcpapi.broker import BeatComputationBroker
from beat.core.utils import setup_logging
logger = logging.getLogger(__name__)
# ----------------------------------------------------------
def onWorkerReady(name):
logger.info("Worker '%s' is ready", name)
try:
worker = Worker.objects.get(name=name)
except Worker.DoesNotExist:
logger.error("No worker named '%s' found in the database", name)
else:
with transaction.atomic():
worker.active = True
worker.info = "Connected to the scheduler"
worker.save()
# ----------------------------------------------------------
def onWorkerGone(name):
logger.info("Worker '%s' is gone", name)
try:
worker = Worker.objects.get(name=name)
except Worker.DoesNotExist:
logger.error("No worker named '%s' found in the database", name)
else:
with transaction.atomic():
worker.active = False
worker.info = "Disconnected from the scheduler"
worker.save()
class Command(BaseCommand):
help = "Start zmq broker"
def add_arguments(self, parser):
parser.add_argument(
"--port",
"-p",
type=int,
dest="port",
default=5555,
help="Port of the broker",
)
def handle(self, *args, **options):
verbosity = int(options["verbosity"])
logger = setup_logging(verbosity, "Broker", __name__)
if verbosity >= 1:
if verbosity == 1:
logger.setLevel(logging.INFO)
elif verbosity >= 2:
logger.setLevel(logging.DEBUG)
logger.info("starting broker")
address = "tcp://*:{}".format(options["port"])
broker = BeatComputationBroker(verbosity >= 2)
broker.set_worker_callbacks(onWorkerReady, onWorkerGone)
broker.bind(address)
broker.mediate()
broker.purge_workers()
logger.info("broker stopped")
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# encoding: utf-8
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import logging
import multiprocessing
import signal
from django.core.management.base import BaseCommand
from django.core.management import call_command
from django.conf import settings
from django import db
from beat.core.utils import find_free_port
logger = logging.getLogger(__name__)
def start_broker(port, verbosity=0):
db.connections.close_all()
call_command("broker", port=port, verbosity=verbosity)
def start_scheduler(broker_address, verbosity=0):
db.connections.close_all()
call_command("scheduler", broker_address=broker_address, verbosity=verbosity)
def start_worker(broker_address, prefix, cache, use_docker, verbosity=0):
call_command(
"worker",
broker_address=broker_address,
prefix=prefix,
cache=cache,
use_docker=use_docker,
verbosity=verbosity,
)
class Command(BaseCommand):
help = "Run a complete local scheduler/broker/worker setup"
def __init__(self):
super(Command, self).__init__()
self.broker = None
self.worker = None
self.scheduler = None
def __signal_handler(self, signum, frame):
self.scheduler.terminate()
self.worker.terminate()
self.broker.terminate()
def add_arguments(self, parser):
parser.add_argument(
"--docker",
"-d",
action="store_true",
dest="use_docker",
default=False,
help="Use docker",
)
def handle(self, *args, **options):
signal.signal(signal.SIGTERM, self.__signal_handler)
signal.signal(signal.SIGINT, self.__signal_handler)
verbosity = options["verbosity"]
port = find_free_port()
broker_address = "tcp://localhost:{}".format(port)
self.broker = multiprocessing.Process(
target=start_broker, args=(port, verbosity)
)
self.worker = multiprocessing.Process(
target=start_worker,
args=(
broker_address,
settings.PREFIX,
settings.CACHE_ROOT,
options["use_docker"],
verbosity,
),
)
self.scheduler = multiprocessing.Process(
target=start_scheduler, args=(broker_address, verbosity)
)
self.broker.start()
self.worker.start()
self.scheduler.start()
self.broker.join()
self.scheduler.join()
self.worker.join()
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# encoding: utf-8
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import socket
from django.core.management.base import BaseCommand
from django.conf import settings
from beat.core.bcp import worker
from beat.core.utils import setup_logging
class Command(BaseCommand):
help = "Start zmq worker"
def add_arguments(self, parser):
parser.add_argument(
"--name",
"-n",
type=str,
dest="service_name",
default=socket.gethostname(),
help="Service name",
)
parser.add_argument(
"--broker",
"-b",
type=str,
dest="broker_address",
default="tcp://localhost:5555",
help="Broker address",
)
parser.add_argument(
"--docker",
"-d",
action="store_true",
dest="use_docker",
default=False,
help="Use docker",
)
parser.add_argument(
"--prefix", "-p", dest="prefix", default=settings.PREFIX, help="Prefix path"
)
parser.add_argument(
"--cache",
"-c",
type=str,
dest="cache",
default=settings.CACHE_ROOT,
help="Cache path",
)
parser.add_argument(
"--network-name",
dest="network_name",
type=str,
default=None,
help="Docker network name",
)
parser.add_argument(
"--port-range",
dest="port_range",
type=str,
default=None,
help="Docker port range",
)
parser.add_argument(
"--docker-images-cache",
dest="docker_images_cache",
type=str,
default=None,
help="Docker image cache",
)
def handle(self, *args, **options):
verbosity = options["verbosity"]
logger = setup_logging(verbosity, "Worker", __name__)
logger.info("starting worker")
argv = [
"--name=" + options["service_name"],
"--prefix=" + options["prefix"],
"--cache=" + options["cache"],
]
argv.extend(["-v" for i in range(options["verbosity"])])
if options["use_docker"]:
argv.append("--docker")
network_name = options.get("network_name")
if network_name:
argv.append("--docker-network=" + network_name)
port_range = options.get("port_range")
if port_range:
argv.append("--port-range=" + port_range)
docker_images_cache = options.get("docker_images_cache")
if docker_images_cache:
argv.append("--docker-images-cache=" + docker_images_cache)
argv.append(options["broker_address"])
status = worker.main(argv)
logger.info("worker stopped")
return status
This diff is collapsed.
[buildout]
extends = common.cfg
parts += docker
develop = .
[sources]
......@@ -11,3 +12,9 @@ scripts += protractor webdriver-manager
[bower]
base-directory = beat/web
[docker]
recipe = collective.recipe.cmd
cmds = ./bin/python -c 'from beat.core.test.utils import pull_docker_test_images as f; f()'
on_install = true
on_update = true
......@@ -7,6 +7,15 @@ eggs = beat.web
beat.cmdline
beat.core
beat.backend.python
versions = versions
[versions]
django = >=1.11,<2.0
django-rest-swagger = >2.1
django-guardian = >=1.3
djangorestframework = >3.7
django-activity-stream = >= 0.6.5
django-jsonfield = >= 1.0.1
[scripts]
recipe = bob.buildout:scripts
......
......@@ -10,11 +10,10 @@ dependencies:
- beat-devel=2019.03.07
# requirements.txt, they are indirectly pinned through the above
- beat.core
- beat.core=1.8
- docopt
- docutils
- jinja2
- graphviz
- matplotlib
- psutil
- psycopg2
......
[buildout]
extends = common.cfg
always-checkout = force
versions = versions
develop = src/beat.backend.python
src/beat.core
src/beat.cmdline
.
[scripts]
eggs = ${buildout:eggs}
interpreter = python
[sources]
beat.core = git git@gitlab.idiap.ch:beat/beat.core egg=false
beat.cmdline = git git@gitlab.idiap.ch:beat/beat.cmdline egg=false
beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python egg=false
beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false
[bower]
base-directory = ./beat/web/
......@@ -440,13 +440,26 @@ Here is how to do it.
$ ./bin/django runserver
2. Start the full scheduling setup::
$ ./bin/django full_scheduling
This will start all elements of the scheduling/working process. Docker can
be used for the worker node passing the ``--docker`` option.
Each element composing the scheduling can also be started separately:
1. Start a the broker node::
$ ./bin/django broker -v 2
2. Start a single scheduling node::
$ ./bin/scheduler -vv
$ ./bin/django scheduler -v 2
3. Start a worker for your current node::
$ ./bin/worker -vv
$ ./bin/django worker -v 2
By default, the applications are configured to figure out paths and
configuration options by themselves. You can override some defaults via the
......@@ -457,7 +470,7 @@ command line. Just check the output of each of those commands running the
Mixing and matching
===================
You can mix and match any of the above techniques to run a 3-node system
You can mix and match any of the above techniques to run a 4-node system
(all-in-one or discrete) to build a test system to suite to your needs. For
example, it is possible to launch the scheduling activities using the web
server and the page reload trick while launching the worker process separately
......
......@@ -6,9 +6,9 @@ eggs-directory = .buildout/eggs
download-cache = .buildout/download-cache
abi-tag-eggs = true
versions = versions
develop = beat.backend.python
beat.core
beat.cmdline
develop = src/beat.backend.python
src/beat.core
src/beat.cmdline
.
[versions]
......
1.4.2b0
\ No newline at end of file
1.5.0b0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment