Commit 7ab0f93a authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'pyproject_toml' into 'master'

Add pyproject.toml

See merge request !137
parents f84d0710 0a0ccef1
Pipeline #49652 failed with stages
in 174 minutes and 51 seconds
......@@ -6,3 +6,5 @@ use_parentheses=true
ensure_newline_before_comments=true
line_length=88
force_single_line=true
order_by_type=true
lines_between_types=1
......@@ -2,17 +2,17 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/timothycrosley/isort
rev: 5.0.4
rev: 5.8.0
hooks:
- id: isort
files: .*.py
- repo: https://github.com/psf/black
rev: 19.10b0
rev: 20.8b1
hooks:
- id: black
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.1.0
rev: v3.4.0
hooks:
- id: check-ast
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......@@ -26,11 +26,11 @@ repos:
- id: check-yaml
exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/flake8
rev: 3.8.3
rev: 3.9.0
hooks:
- id: flake8
- repo: https://github.com/PyCQA/bandit
rev: 1.6.2
rev: 1.7.0
hooks:
- id: bandit
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......
......@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm):
self._check_language_consistence()
def _check_endpoint_uniqueness(self):
"""Checks for name clashes accross input/output groups
"""
"""Checks for name clashes accross input/output groups"""
all_input_names = []
for group in self.groups:
......@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm):
self._validate_format(type_name, group_name, name, thisformat)
def _validate_required_dataformats(self, dataformat_cache):
"""Makes sure we can load all requested formats
"""
"""Makes sure we can load all requested formats"""
for group in self.groups:
......
......@@ -65,10 +65,12 @@ import os
import signal
import sys
import tempfile
from socket import gethostname
import simplejson as json
import zmq
from docopt import docopt
from ..bcpapi import BCP
......
......@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker
import logging
import signal
import time
from binascii import hexlify
import zmq
......
......@@ -73,8 +73,7 @@ class BeatComputationClient(object):
logger.info("I: connecting to broker at %s...", self.broker)
def send(self, service, request):
"""Send request to broker
"""
"""Send request to broker"""
if not isinstance(request, list):
request = [request]
......
......@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__)
class BeatComputationProcessor(object):
"""BEAT Computation Protocol Processor API, Python version
"""
"""BEAT Computation Protocol Processor API, Python version"""
def __init__(self, poller, address, verbose=False):
self.verbose = verbose
......
......@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__)
class BeatComputationWorker(object):
"""BEAT Computation Protocol Worker API, Python version
"""
"""BEAT Computation Protocol Worker API, Python version"""
HEARTBEAT_LIVENESS = 3 # 3-5 is reasonable
......
......@@ -51,6 +51,7 @@ import time
import docker
import simplejson as json
from packaging import version
from beat.core import stats
......@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__)
class Host(object):
"""An object of this class can connect to the docker host and resolve stuff
"""
"""An object of this class can connect to the docker host and resolve stuff"""
images_cache = {}
......@@ -807,19 +807,17 @@ class Container:
self._temporary_filesystems[path] = size
def set_name(self, name):
""" Set the name to be used by the container in place of the docker
"""Set the name to be used by the container in place of the docker
auto generated one.
"""
self._name = name
def set_workdir(self, workdir):
""" Set the work folder to be used by the container
"""
"""Set the work folder to be used by the container"""
self._workdir = workdir
def set_entrypoint(self, entrypoint):
""" Set the entry point to be used by the container
"""
"""Set the entry point to be used by the container"""
self._entrypoint = entrypoint
def add_volume(self, path, mount_path, read_only=True):
......@@ -952,8 +950,7 @@ class Container:
@property
def environment_variables(self):
"""Returns the environment variables to set on this container.
"""
"""Returns the environment variables to set on this container."""
environment_variables = []
for k, v in self._environment_variables.items():
......
......@@ -302,8 +302,7 @@ class BaseExecutor(object):
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Closes all sinks and disconnects inputs and outputs
"""
"""Closes all sinks and disconnects inputs and outputs"""
for sink in self.data_sinks:
# we save the output only if no valid error has been thrown
......
......@@ -44,6 +44,7 @@ Execution utilities
import logging
import os
import shutil
from collections import namedtuple
import requests
......@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor):
databases_infos = {}
for db_name, db_object, in self.databases.items():
for (
db_name,
db_object,
) in self.databases.items():
json_path = os.path.join(root_folder, db_name + ".json")
with open(json_path, "r") as f:
......
......@@ -86,23 +86,20 @@ def hashBlockOutput(
# Note: 'block_name' and 'algorithm_name' aren't used to compute the hash,
# but are useful when an application wants to implement its own hash
# function
s = (
_compact(
"""{
s = _compact(
"""{
"algorithm": "%s",
"parameters": %s,
"environment": %s,
"inputs": %s,
"output": "%s"
}"""
)
% (
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
output_name,
)
) % (
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
output_name,
)
return hash(s)
......@@ -131,23 +128,20 @@ def hashAnalyzer(
# Note: 'analyzer_name' isn't used to compute the hash, but is useful when
# an applications want to implement its own hash function
s = (
_compact(
"""{
s = _compact(
"""{
"algorithm_name": "%s",
"algorithm": "%s",
"parameters": %s,
"environment": %s,
"inputs": %s
}"""
)
% (
algorithm_name,
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
)
) % (
algorithm_name,
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
)
return hash(s)
......
......@@ -277,8 +277,7 @@ class Plotter(object):
_check_language_consistence = algorithm.Algorithm._check_language_consistence
def _validate_dataformat(self, dataformat_cache):
"""Makes sure we can load the requested format
"""
"""Makes sure we can load the requested format"""
name = self.data["dataformat"]
......@@ -304,8 +303,7 @@ class Plotter(object):
@property
def name(self):
"""The name of this object
"""
"""The name of this object"""
return self._name or "__unnamed_plotter__"
@name.setter
......
......@@ -7,7 +7,12 @@ from beat.backend.python.database import View
class FooView(View):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -67,25 +67,25 @@ def load_schema(schema_name, version=1):
"""Returns a JSON validator for the schema given the relative name
Parameters:
Parameters:
schema_name (str): the name of the schema to load. This value corresponds
to the filename inside our schema directory (where this file is located)
and should *exclude* the extension ``.json``.
schema_name (str): the name of the schema to load. This value corresponds
to the filename inside our schema directory (where this file is located)
and should *exclude* the extension ``.json``.
version (int): the version of the schema to use.
version (int): the version of the schema to use.
Returns:
Returns:
jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
Raises:
Raises:
jsonschema.SchemaError: If there is an error on the schema.
jsonschema.SchemaError: If there is an error on the schema.
"""
"""
fname = pkg_resources.resource_filename(
__name__, os.path.join(schema_name, "%d.json" % version)
......@@ -109,54 +109,54 @@ def load_schema(schema_name, version=1):
def validate(schema_name, data):
"""Validates the input data using the schema
This function handles schema versionning in the context of BEAT transparently
by first peeking the schema version required by the JSON data and then
validating the JSON data against the proper schema version for the respective
type.
This function handles schema versionning in the context of BEAT transparently
by first peeking the schema version required by the JSON data and then
validating the JSON data against the proper schema version for the respective
type.
Example:
Example:
.. code-block:: python
.. code-block:: python
try:
cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
except json.JSONDecodeError as e:
print(e)
try:
cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
except json.JSONDecodeError as e:
print(e)
Parameters:
Parameters:
schema_name (str): The relative path to the schema to use for validation.
For example, to validate a toolchain, use ``'toolchain'``.
schema_name (str): The relative path to the schema to use for validation.
For example, to validate a toolchain, use ``'toolchain'``.
data (object, str, file): The piece of data to validate. The input can be a
valid python object that represents a JSON structure, a file, from which
the JSON contents will be read out or a string.
data (object, str, file): The piece of data to validate. The input can be a
valid python object that represents a JSON structure, a file, from which
the JSON contents will be read out or a string.
If ``data`` is a string and represents a valid filesystem path, the
relevant file will be opened and read as with
:py:func:`json.load``. Otherwise, it will be considered to be
string containing a valid JSON structure that will be loaded as with
:py:func:`json.loads`.
If ``data`` is a string and represents a valid filesystem path, the
relevant file will be opened and read as with
:py:func:`json.load``. Otherwise, it will be considered to be
string containing a valid JSON structure that will be loaded as with
:py:func:`json.loads`.
Note that if the file is opened and read internally using
:py:func:`json.load`, exceptions may be thrown by that subsystem,
concerning the file structure. Consult the manual page for
:py:mod:`simplejson` for details.
Note that if the file is opened and read internally using
:py:func:`json.load`, exceptions may be thrown by that subsystem,
concerning the file structure. Consult the manual page for
:py:mod:`simplejson` for details.
Returns:
Returns:
A tuple with two elements: the cleaned JSON data structure, after
processing and a list of errors found by ``jsonschema``. If no errors
occur, then returns an empty list for the second element of the tuple.
A tuple with two elements: the cleaned JSON data structure, after
processing and a list of errors found by ``jsonschema``. If no errors
occur, then returns an empty list for the second element of the tuple.
Raises:
Raises:
jsonschema.SchemaError: If there is an error on the schema.
jsonschema.SchemaError: If there is an error on the schema.
"""
"""
try:
data = maybe_load_json(data)
......
......@@ -53,9 +53,11 @@ Options:
import copy
import os
import sys
from collections import OrderedDict
import simplejson as json
from docopt import docopt
from ..database import Database
......
......@@ -72,6 +72,7 @@ except ImportError:
import queue as Queue
import tempfile
from socket import gethostname
from docopt import docopt
......@@ -84,7 +85,7 @@ from ..version import __version__
from ..worker import WorkerController
stop = False
logger = None
logger = logging.getLogger(__name__)
# ----------------------------------------------------------
......@@ -250,9 +251,6 @@ def main(user_input=None):
beat_core_logger.setLevel(logging.WARNING)
beat_backend_logger.setLevel(logging.WARNING)
global logger
logger = logging.getLogger(__name__)
# Check the prefix path
prefix = args["--prefix"] if args["--prefix"] is not None else "."
if not os.path.exists(prefix):
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -37,12 +37,14 @@
# Tests for experiment execution
import multiprocessing
import multiprocessing.queues
import os
import queue
import unittest
import simplejson as json
import zmq
from flaky import flaky
from ..bcp import broker
......@@ -134,24 +136,37 @@ DEFAULT_MAX_ITERATION_COUNT = 30
class ZMQBrokerProcess(multiprocessing.Process):
def __init__(self, port, verbose, callbacks=None):
def __init__(self, port, verbose, queue=None):
super(ZMQBrokerProcess, self).__init__()
self.port = port
self.verbose = verbose
self.callbacks = callbacks
self.queue = queue
def run(self):
return broker.run(self.port, verbose=self.verbose, callbacks=self.callbacks)
callbacks = None
if self.queue:
callbacks = self.queue.callbacks()
return broker.run(self.port, verbose=self.verbose, callbacks=callbacks)
class ZMQWorkerProcess(multiprocessing.Process):
def __init__(
self, address, name, verbose, use_docker=False, docker_images_cache=None
self,
address,
name,
verbose,
prefix,
tmp_prefix,
use_docker=False,
docker_images_cache=None,
):
super(ZMQWorkerProcess, self).__init__()
self.broker_address = address
self.service_name = name
self.verbose = verbose
self.prefix = prefix
self.tmp_prefix = tmp_prefix
self.use_docker = use_docker
self.docker_images_cache = None
......@@ -160,8 +175,8 @@ class ZMQWorkerProcess(multiprocessing.Process):
self.broker_address,
service_name=self.service_name,
verbose=self.verbose,
prefix=prefix,
cache=tmp_prefix,
prefix=self.prefix,
cache=self.tmp_prefix,
use_docker=self.use_docker,
docker_images_cache=self.docker_images_cache,
)
......@@ -183,15 +198,27 @@ class ExecutionTestCase(unittest.TestCase):
view.index(os.path.join(tmp_prefix, input_cfg["path"]))
class TestBroker(unittest.TestCase):
class CallbackedQueue(multiprocessing.queues.Queue):
def __init__(self, *args, **kwargs):
from multiprocessing.context import BaseContext
ctx = BaseContext()
ctx._name = "callbacks"
super().__init__(*args, **kwargs, ctx=ctx)
def __on_ready(self, name):
self.queue.put("ready")
self.put("ready")
def __on_gone(self, name):
self.queue.put("gone")
self.put("gone")
def callbacks(self):
return (self.__on_ready, self.__on_gone)
class TestBroker(unittest.TestCase):
def setUp(self):
self.queue = multiprocessing.Queue()
self.queue = CallbackedQueue()
def test_callback(self):
worker_name = b"test_worker"
......@@ -199,12 +226,12 @@ class TestBroker(unittest.TestCase):
port = find_free_port()
broker_address = "tcp://localhost:{}".format(port)
broker_p = ZMQBrokerProcess(
port, VERBOSE_BCP_LOGGING, (self.__on_ready, self.__on_gone)
)
broker_p = ZMQBrokerProcess(port, VERBOSE_BCP_LOGGING, self.queue)
broker_p.start()
worker = ZMQWorkerProcess(broker_address, worker_name, VERBOSE_BCP_LOGGING)
worker = ZMQWorkerProcess(
broker_address, worker_name, VERBOSE_BCP_LOGGING, prefix, tmp_prefix
)
worker.start()
worker.join(2) # Give the worker enough time to announce itself
worker.terminate()
......@@ -248,6 +275,8 @@ class TestBCP(ExecutionTestCase):
broker_address,
self.worker_name,
VERBOSE_BCP_LOGGING,
prefix,
tmp_prefix,
self.use_docker,
self.docker_images_cache,
)
......
......@@ -39,6 +39,7 @@ import os
import tempfile
import time
import unittest
from tempfile import TemporaryDirectory
import pkg_resources
......
......@@ -306,7 +306,8 @@ class TestDockerExecution(BaseExecutionMixIn):
def test_loop_mix_db_env_error(self):
with nose.tools.assert_raises(RuntimeError) as context:
self.execute(
"errors/user/loop/1/loop_mix_db_env", [None],
"errors/user/loop/1/loop_mix_db_env",
[None],
)
nose.tools.assert_true(
......@@ -317,7 +318,8 @@ class TestDockerExecution(BaseExecutionMixIn):
def test_loop_two_db_env_error(self):
with nose.tools.assert_raises(RuntimeError) as context:
self.execute(