Merge branch 'pyproject_toml' into 'master'

Add pyproject.toml

See merge request !137

.isort.cfg

@@ -6,3 +6,5 @@ use_parentheses=true
 ensure_newline_before_comments=true
 line_length=88
 force_single_line=true
+order_by_type=true
+lines_between_types=1

.pre-commit-config.yaml

@@ -2,17 +2,17 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.0.4
+    rev: 5.8.0
     hooks:
       - id: isort
         files: .*.py
   - repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
       - id: black
         exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
       - id: check-ast
         exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
@@ -26,11 +26,11 @@ repos:
       - id: check-yaml
         exclude: conda/meta.yaml
   - repo: https://github.com/PyCQA/flake8
-    rev: 3.8.3
+    rev: 3.9.0
     hooks:
       - id: flake8
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.6.2
+    rev: 1.7.0
     hooks:
       - id: bandit
         exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py

beat/core/algorithm.py

@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm):
         self._check_language_consistence()
 
     def _check_endpoint_uniqueness(self):
-        """Checks for name clashes accross input/output groups
-        """
+        """Checks for name clashes accross input/output groups"""
 
         all_input_names = []
         for group in self.groups:
@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm):
             self._validate_format(type_name, group_name, name, thisformat)
 
     def _validate_required_dataformats(self, dataformat_cache):
-        """Makes sure we can load all requested formats
-        """
+        """Makes sure we can load all requested formats"""
 
         for group in self.groups:
 

beat/core/bcp/worker.py

@@ -65,10 +65,12 @@ import os
 import signal
 import sys
 import tempfile
+
 from socket import gethostname
 
 import simplejson as json
 import zmq
+
 from docopt import docopt
 
 from ..bcpapi import BCP

beat/core/bcpapi/broker.py

@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker
 import logging
 import signal
 import time
+
 from binascii import hexlify
 
 import zmq

beat/core/bcpapi/client.py

@@ -73,8 +73,7 @@ class BeatComputationClient(object):
             logger.info("I: connecting to broker at %s...", self.broker)
 
     def send(self, service, request):
-        """Send request to broker
-        """
+        """Send request to broker"""
         if not isinstance(request, list):
             request = [request]
 

beat/core/bcpapi/processor.py

@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__)
 
 
 class BeatComputationProcessor(object):
-    """BEAT Computation Protocol Processor API, Python version
-    """
+    """BEAT Computation Protocol Processor API, Python version"""
 
     def __init__(self, poller, address, verbose=False):
         self.verbose = verbose

beat/core/bcpapi/worker.py

@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__)
 
 
 class BeatComputationWorker(object):
-    """BEAT Computation Protocol Worker API, Python version
-    """
+    """BEAT Computation Protocol Worker API, Python version"""
 
     HEARTBEAT_LIVENESS = 3  # 3-5 is reasonable
 

beat/core/dock.py

@@ -51,6 +51,7 @@ import time
 
 import docker
 import simplejson as json
+
 from packaging import version
 
 from beat.core import stats
@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__)
 
 
 class Host(object):
-    """An object of this class can connect to the docker host and resolve stuff
-    """
+    """An object of this class can connect to the docker host and resolve stuff"""
 
     images_cache = {}
 
@@ -807,19 +807,17 @@ class Container:
                 self._temporary_filesystems[path] = size
 
     def set_name(self, name):
-        """ Set the name to be used by the container in place of the docker
+        """Set the name to be used by the container in place of the docker
         auto generated one.
         """
         self._name = name
 
     def set_workdir(self, workdir):
-        """ Set the work folder to be used by the container
-        """
+        """Set the work folder to be used by the container"""
         self._workdir = workdir
 
     def set_entrypoint(self, entrypoint):
-        """ Set the entry point to be used by the container
-        """
+        """Set the entry point to be used by the container"""
         self._entrypoint = entrypoint
 
     def add_volume(self, path, mount_path, read_only=True):
@@ -952,8 +950,7 @@ class Container:
 
     @property
     def environment_variables(self):
-        """Returns the environment variables to set on this container.
-        """
+        """Returns the environment variables to set on this container."""
 
         environment_variables = []
         for k, v in self._environment_variables.items():

beat/core/execution/base.py

@@ -302,8 +302,7 @@ class BaseExecutor(object):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        """Closes all sinks and disconnects inputs and outputs
-        """
+        """Closes all sinks and disconnects inputs and outputs"""
 
         for sink in self.data_sinks:
             # we save the output only if no valid error has been thrown

beat/core/execution/docker.py

@@ -44,6 +44,7 @@ Execution utilities
 import logging
 import os
 import shutil
+
 from collections import namedtuple
 
 import requests
@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor):
 
         databases_infos = {}
 
-        for db_name, db_object, in self.databases.items():
+        for (
+            db_name,
+            db_object,
+        ) in self.databases.items():
             json_path = os.path.join(root_folder, db_name + ".json")
 
             with open(json_path, "r") as f:

beat/core/hash.py

@@ -86,23 +86,20 @@ def hashBlockOutput(
     # Note: 'block_name' and 'algorithm_name' aren't used to compute the hash,
     # but are useful when an application wants to implement its own hash
     # function
-    s = (
-        _compact(
-            """{
+    s = _compact(
+        """{
         "algorithm": "%s",
         "parameters": %s,
         "environment": %s,
         "inputs": %s,
         "output": "%s"
 }"""
-        )
-        % (
-            algorithm_hash,
-            _stringify(parameters),
-            _stringify(environment),
-            _stringify(input_hashes),
-            output_name,
-        )
+    ) % (
+        algorithm_hash,
+        _stringify(parameters),
+        _stringify(environment),
+        _stringify(input_hashes),
+        output_name,
     )
     return hash(s)
 
@@ -131,23 +128,20 @@ def hashAnalyzer(
 
     # Note: 'analyzer_name' isn't used to compute the hash, but is useful when
     # an applications want to implement its own hash function
-    s = (
-        _compact(
-            """{
+    s = _compact(
+        """{
         "algorithm_name": "%s",
         "algorithm": "%s",
         "parameters": %s,
         "environment": %s,
         "inputs": %s
 }"""
-        )
-        % (
-            algorithm_name,
-            algorithm_hash,
-            _stringify(parameters),
-            _stringify(environment),
-            _stringify(input_hashes),
-        )
+    ) % (
+        algorithm_name,
+        algorithm_hash,
+        _stringify(parameters),
+        _stringify(environment),
+        _stringify(input_hashes),
     )
     return hash(s)
 

beat/core/plotter.py

@@ -277,8 +277,7 @@ class Plotter(object):
         _check_language_consistence = algorithm.Algorithm._check_language_consistence
 
     def _validate_dataformat(self, dataformat_cache):
-        """Makes sure we can load the requested format
-        """
+        """Makes sure we can load the requested format"""
 
         name = self.data["dataformat"]
 
@@ -304,8 +303,7 @@ class Plotter(object):
 
     @property
     def name(self):
-        """The name of this object
-        """
+        """The name of this object"""
         return self._name or "__unnamed_plotter__"
 
     @name.setter

beat/core/prototypes/database.py

@@ -7,7 +7,12 @@ from beat.backend.python.database import View
 
 class FooView(View):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/schema/__init__.py

@@ -67,25 +67,25 @@ def load_schema(schema_name, version=1):
     """Returns a JSON validator for the schema given the relative name
 
 
-  Parameters:
+    Parameters:
 
-    schema_name (str): the name of the schema to load. This value corresponds
-      to the filename inside our schema directory (where this file is located)
-      and should *exclude* the extension ``.json``.
+      schema_name (str): the name of the schema to load. This value corresponds
+        to the filename inside our schema directory (where this file is located)
+        and should *exclude* the extension ``.json``.
 
-    version (int): the version of the schema to use.
+      version (int): the version of the schema to use.
 
 
-  Returns:
+    Returns:
 
-    jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
+      jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
 
 
-  Raises:
+    Raises:
 
-    jsonschema.SchemaError: If there is an error on the schema.
+      jsonschema.SchemaError: If there is an error on the schema.
 
-  """
+    """
 
     fname = pkg_resources.resource_filename(
         __name__, os.path.join(schema_name, "%d.json" % version)
@@ -109,54 +109,54 @@ def load_schema(schema_name, version=1):
 def validate(schema_name, data):
     """Validates the input data using the schema
 
-  This function handles schema versionning in the context of BEAT transparently
-  by first peeking the schema version required by the JSON data and then
-  validating the JSON data against the proper schema version for the respective
-  type.
+    This function handles schema versionning in the context of BEAT transparently
+    by first peeking the schema version required by the JSON data and then
+    validating the JSON data against the proper schema version for the respective
+    type.
 
 
-  Example:
+    Example:
 
-    .. code-block:: python
+      .. code-block:: python
 
-       try:
-           cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
-       except json.JSONDecodeError as e:
-           print(e)
+         try:
+             cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
+         except json.JSONDecodeError as e:
+             print(e)
 
 
-  Parameters:
+    Parameters:
 
-    schema_name (str): The relative path to the schema to use for validation.
-      For example, to validate a toolchain, use ``'toolchain'``.
+      schema_name (str): The relative path to the schema to use for validation.
+        For example, to validate a toolchain, use ``'toolchain'``.
 
-    data (object, str, file): The piece of data to validate. The input can be a
-      valid python object that represents a JSON structure, a file, from which
-      the JSON contents will be read out or a string.
+      data (object, str, file): The piece of data to validate. The input can be a
+        valid python object that represents a JSON structure, a file, from which
+        the JSON contents will be read out or a string.
 
-      If ``data`` is a string and represents a valid filesystem path, the
-      relevant file will be opened and read as with
-      :py:func:`json.load``. Otherwise, it will be considered to be
-      string containing a valid JSON structure that will be loaded as with
-      :py:func:`json.loads`.
+        If ``data`` is a string and represents a valid filesystem path, the
+        relevant file will be opened and read as with
+        :py:func:`json.load``. Otherwise, it will be considered to be
+        string containing a valid JSON structure that will be loaded as with
+        :py:func:`json.loads`.
 
-      Note that if the file is opened and read internally using
-      :py:func:`json.load`, exceptions may be thrown by that subsystem,
-      concerning the file structure. Consult the manual page for
-      :py:mod:`simplejson` for details.
+        Note that if the file is opened and read internally using
+        :py:func:`json.load`, exceptions may be thrown by that subsystem,
+        concerning the file structure. Consult the manual page for
+        :py:mod:`simplejson` for details.
 
 
-  Returns:
+    Returns:
 
-    A tuple with two elements: the cleaned JSON data structure, after
-    processing and a list of errors found by ``jsonschema``. If no errors
-    occur, then returns an empty list for the second element of the tuple.
+      A tuple with two elements: the cleaned JSON data structure, after
+      processing and a list of errors found by ``jsonschema``. If no errors
+      occur, then returns an empty list for the second element of the tuple.
 
-  Raises:
+    Raises:
 
-    jsonschema.SchemaError: If there is an error on the schema.
+      jsonschema.SchemaError: If there is an error on the schema.
 
-  """
+    """
 
     try:
         data = maybe_load_json(data)

beat/core/scripts/migrate_db_to_v2.py

@@ -53,9 +53,11 @@ Options:
 import copy
 import os
 import sys
+
 from collections import OrderedDict
 
 import simplejson as json
+
 from docopt import docopt
 
 from ..database import Database

beat/core/scripts/worker.py

@@ -72,6 +72,7 @@ except ImportError:
     import queue as Queue
 
 import tempfile
+
 from socket import gethostname
 
 from docopt import docopt
@@ -84,7 +85,7 @@ from ..version import __version__
 from ..worker import WorkerController
 
 stop = False
-logger = None
+logger = logging.getLogger(__name__)
 
 
 # ----------------------------------------------------------
@@ -250,9 +251,6 @@ def main(user_input=None):
         beat_core_logger.setLevel(logging.WARNING)
         beat_backend_logger.setLevel(logging.WARNING)
 
-    global logger
-    logger = logging.getLogger(__name__)
-
     # Check the prefix path
     prefix = args["--prefix"] if args["--prefix"] is not None else "."
     if not os.path.exists(prefix):

beat/core/test/prefix/databases/simple/1.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple/2.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_env/1.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_env/2.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_invalid_env/1.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_invalid_env/2.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_rawdata_access/1.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/prefix/databases/simple_rawdata_access/2.py

@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
 
 class View(BaseView):
     def setup(
-        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        start_index=None,
+        end_index=None,
     ):
         """Initializes the database"""
 

beat/core/test/test_bcp.py

@@ -37,12 +37,14 @@
 # Tests for experiment execution
 
 import multiprocessing
+import multiprocessing.queues
 import os
 import queue
 import unittest
 
 import simplejson as json
 import zmq
+
 from flaky import flaky
 
 from ..bcp import broker
@@ -134,24 +136,37 @@ DEFAULT_MAX_ITERATION_COUNT = 30
 
 
 class ZMQBrokerProcess(multiprocessing.Process):
-    def __init__(self, port, verbose, callbacks=None):
+    def __init__(self, port, verbose, queue=None):
         super(ZMQBrokerProcess, self).__init__()
         self.port = port
         self.verbose = verbose
-        self.callbacks = callbacks
+        self.queue = queue
 
     def run(self):
-        return broker.run(self.port, verbose=self.verbose, callbacks=self.callbacks)
+        callbacks = None
+        if self.queue:
+            callbacks = self.queue.callbacks()
+
+        return broker.run(self.port, verbose=self.verbose, callbacks=callbacks)
 
 
 class ZMQWorkerProcess(multiprocessing.Process):
     def __init__(
-        self, address, name, verbose, use_docker=False, docker_images_cache=None
+        self,
+        address,
+        name,
+        verbose,
+        prefix,
+        tmp_prefix,
+        use_docker=False,
+        docker_images_cache=None,
     ):
         super(ZMQWorkerProcess, self).__init__()
         self.broker_address = address
         self.service_name = name
         self.verbose = verbose
+        self.prefix = prefix
+        self.tmp_prefix = tmp_prefix
         self.use_docker = use_docker
         self.docker_images_cache = None
 
@@ -160,8 +175,8 @@ class ZMQWorkerProcess(multiprocessing.Process):
             self.broker_address,
             service_name=self.service_name,
             verbose=self.verbose,
-            prefix=prefix,
-            cache=tmp_prefix,
+            prefix=self.prefix,
+            cache=self.tmp_prefix,
             use_docker=self.use_docker,
             docker_images_cache=self.docker_images_cache,
         )
@@ -183,15 +198,27 @@ class ExecutionTestCase(unittest.TestCase):
             view.index(os.path.join(tmp_prefix, input_cfg["path"]))
 
 
-class TestBroker(unittest.TestCase):
+class CallbackedQueue(multiprocessing.queues.Queue):
+    def __init__(self, *args, **kwargs):
+        from multiprocessing.context import BaseContext
+
+        ctx = BaseContext()
+        ctx._name = "callbacks"
+        super().__init__(*args, **kwargs, ctx=ctx)
+
     def __on_ready(self, name):
-        self.queue.put("ready")
+        self.put("ready")
 
     def __on_gone(self, name):
-        self.queue.put("gone")
+        self.put("gone")
+
+    def callbacks(self):
+        return (self.__on_ready, self.__on_gone)
 
+
+class TestBroker(unittest.TestCase):
     def setUp(self):
-        self.queue = multiprocessing.Queue()
+        self.queue = CallbackedQueue()
 
     def test_callback(self):
         worker_name = b"test_worker"
@@ -199,12 +226,12 @@ class TestBroker(unittest.TestCase):
         port = find_free_port()
         broker_address = "tcp://localhost:{}".format(port)
 
-        broker_p = ZMQBrokerProcess(
-            port, VERBOSE_BCP_LOGGING, (self.__on_ready, self.__on_gone)
-        )
+        broker_p = ZMQBrokerProcess(port, VERBOSE_BCP_LOGGING, self.queue)
         broker_p.start()
 
-        worker = ZMQWorkerProcess(broker_address, worker_name, VERBOSE_BCP_LOGGING)
+        worker = ZMQWorkerProcess(
+            broker_address, worker_name, VERBOSE_BCP_LOGGING, prefix, tmp_prefix
+        )
         worker.start()
         worker.join(2)  # Give the worker enough time to announce itself
         worker.terminate()
@@ -248,6 +275,8 @@ class TestBCP(ExecutionTestCase):
             broker_address,
             self.worker_name,
             VERBOSE_BCP_LOGGING,
+            prefix,
+            tmp_prefix,
             self.use_docker,
             self.docker_images_cache,
         )

beat/core/test/test_docker.py

@@ -39,6 +39,7 @@ import os
 import tempfile
 import time
 import unittest
+
 from tempfile import TemporaryDirectory
 
 import pkg_resources

beat/core/test/test_docker_execution.py

@@ -306,7 +306,8 @@ class TestDockerExecution(BaseExecutionMixIn):
     def test_loop_mix_db_env_error(self):
         with nose.tools.assert_raises(RuntimeError) as context:
             self.execute(
-                "errors/user/loop/1/loop_mix_db_env", [None],
+                "errors/user/loop/1/loop_mix_db_env",
+                [None],
             )
 
         nose.tools.assert_true(
@@ -317,7 +318,8 @@ class TestDockerExecution(BaseExecutionMixIn):
     def test_loop_two_db_env_error(self):
         with nose.tools.assert_raises(RuntimeError) as context:
             self.execute(