Commit 7ab0f93a authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'pyproject_toml' into 'master'

Add pyproject.toml

See merge request !137
parents f84d0710 0a0ccef1
Pipeline #49652 failed with stages
in 174 minutes and 51 seconds
...@@ -6,3 +6,5 @@ use_parentheses=true ...@@ -6,3 +6,5 @@ use_parentheses=true
ensure_newline_before_comments=true ensure_newline_before_comments=true
line_length=88 line_length=88
force_single_line=true force_single_line=true
order_by_type=true
lines_between_types=1
...@@ -2,17 +2,17 @@ ...@@ -2,17 +2,17 @@
# See https://pre-commit.com/hooks.html for more hooks # See https://pre-commit.com/hooks.html for more hooks
repos: repos:
- repo: https://github.com/timothycrosley/isort - repo: https://github.com/timothycrosley/isort
rev: 5.0.4 rev: 5.8.0
hooks: hooks:
- id: isort - id: isort
files: .*.py files: .*.py
- repo: https://github.com/psf/black - repo: https://github.com/psf/black
rev: 19.10b0 rev: 20.8b1
hooks: hooks:
- id: black - id: black
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.1.0 rev: v3.4.0
hooks: hooks:
- id: check-ast - id: check-ast
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
...@@ -26,11 +26,11 @@ repos: ...@@ -26,11 +26,11 @@ repos:
- id: check-yaml - id: check-yaml
exclude: conda/meta.yaml exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/flake8 - repo: https://github.com/PyCQA/flake8
rev: 3.8.3 rev: 3.9.0
hooks: hooks:
- id: flake8 - id: flake8
- repo: https://github.com/PyCQA/bandit - repo: https://github.com/PyCQA/bandit
rev: 1.6.2 rev: 1.7.0
hooks: hooks:
- id: bandit - id: bandit
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......
...@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm): ...@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm):
self._check_language_consistence() self._check_language_consistence()
def _check_endpoint_uniqueness(self): def _check_endpoint_uniqueness(self):
"""Checks for name clashes accross input/output groups """Checks for name clashes accross input/output groups"""
"""
all_input_names = [] all_input_names = []
for group in self.groups: for group in self.groups:
...@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm): ...@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm):
self._validate_format(type_name, group_name, name, thisformat) self._validate_format(type_name, group_name, name, thisformat)
def _validate_required_dataformats(self, dataformat_cache): def _validate_required_dataformats(self, dataformat_cache):
"""Makes sure we can load all requested formats """Makes sure we can load all requested formats"""
"""
for group in self.groups: for group in self.groups:
......
...@@ -65,10 +65,12 @@ import os ...@@ -65,10 +65,12 @@ import os
import signal import signal
import sys import sys
import tempfile import tempfile
from socket import gethostname from socket import gethostname
import simplejson as json import simplejson as json
import zmq import zmq
from docopt import docopt from docopt import docopt
from ..bcpapi import BCP from ..bcpapi import BCP
......
...@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker ...@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker
import logging import logging
import signal import signal
import time import time
from binascii import hexlify from binascii import hexlify
import zmq import zmq
......
...@@ -73,8 +73,7 @@ class BeatComputationClient(object): ...@@ -73,8 +73,7 @@ class BeatComputationClient(object):
logger.info("I: connecting to broker at %s...", self.broker) logger.info("I: connecting to broker at %s...", self.broker)
def send(self, service, request): def send(self, service, request):
"""Send request to broker """Send request to broker"""
"""
if not isinstance(request, list): if not isinstance(request, list):
request = [request] request = [request]
......
...@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__) ...@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__)
class BeatComputationProcessor(object): class BeatComputationProcessor(object):
"""BEAT Computation Protocol Processor API, Python version """BEAT Computation Protocol Processor API, Python version"""
"""
def __init__(self, poller, address, verbose=False): def __init__(self, poller, address, verbose=False):
self.verbose = verbose self.verbose = verbose
......
...@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__) ...@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__)
class BeatComputationWorker(object): class BeatComputationWorker(object):
"""BEAT Computation Protocol Worker API, Python version """BEAT Computation Protocol Worker API, Python version"""
"""
HEARTBEAT_LIVENESS = 3 # 3-5 is reasonable HEARTBEAT_LIVENESS = 3 # 3-5 is reasonable
......
...@@ -51,6 +51,7 @@ import time ...@@ -51,6 +51,7 @@ import time
import docker import docker
import simplejson as json import simplejson as json
from packaging import version from packaging import version
from beat.core import stats from beat.core import stats
...@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__) ...@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__)
class Host(object): class Host(object):
"""An object of this class can connect to the docker host and resolve stuff """An object of this class can connect to the docker host and resolve stuff"""
"""
images_cache = {} images_cache = {}
...@@ -807,19 +807,17 @@ class Container: ...@@ -807,19 +807,17 @@ class Container:
self._temporary_filesystems[path] = size self._temporary_filesystems[path] = size
def set_name(self, name): def set_name(self, name):
""" Set the name to be used by the container in place of the docker """Set the name to be used by the container in place of the docker
auto generated one. auto generated one.
""" """
self._name = name self._name = name
def set_workdir(self, workdir): def set_workdir(self, workdir):
""" Set the work folder to be used by the container """Set the work folder to be used by the container"""
"""
self._workdir = workdir self._workdir = workdir
def set_entrypoint(self, entrypoint): def set_entrypoint(self, entrypoint):
""" Set the entry point to be used by the container """Set the entry point to be used by the container"""
"""
self._entrypoint = entrypoint self._entrypoint = entrypoint
def add_volume(self, path, mount_path, read_only=True): def add_volume(self, path, mount_path, read_only=True):
...@@ -952,8 +950,7 @@ class Container: ...@@ -952,8 +950,7 @@ class Container:
@property @property
def environment_variables(self): def environment_variables(self):
"""Returns the environment variables to set on this container. """Returns the environment variables to set on this container."""
"""
environment_variables = [] environment_variables = []
for k, v in self._environment_variables.items(): for k, v in self._environment_variables.items():
......
...@@ -302,8 +302,7 @@ class BaseExecutor(object): ...@@ -302,8 +302,7 @@ class BaseExecutor(object):
return self return self
def __exit__(self, exc_type, exc_value, traceback): def __exit__(self, exc_type, exc_value, traceback):
"""Closes all sinks and disconnects inputs and outputs """Closes all sinks and disconnects inputs and outputs"""
"""
for sink in self.data_sinks: for sink in self.data_sinks:
# we save the output only if no valid error has been thrown # we save the output only if no valid error has been thrown
......
...@@ -44,6 +44,7 @@ Execution utilities ...@@ -44,6 +44,7 @@ Execution utilities
import logging import logging
import os import os
import shutil import shutil
from collections import namedtuple from collections import namedtuple
import requests import requests
...@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor): ...@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor):
databases_infos = {} databases_infos = {}
for db_name, db_object, in self.databases.items(): for (
db_name,
db_object,
) in self.databases.items():
json_path = os.path.join(root_folder, db_name + ".json") json_path = os.path.join(root_folder, db_name + ".json")
with open(json_path, "r") as f: with open(json_path, "r") as f:
......
...@@ -86,23 +86,20 @@ def hashBlockOutput( ...@@ -86,23 +86,20 @@ def hashBlockOutput(
# Note: 'block_name' and 'algorithm_name' aren't used to compute the hash, # Note: 'block_name' and 'algorithm_name' aren't used to compute the hash,
# but are useful when an application wants to implement its own hash # but are useful when an application wants to implement its own hash
# function # function
s = ( s = _compact(
_compact( """{
"""{
"algorithm": "%s", "algorithm": "%s",
"parameters": %s, "parameters": %s,
"environment": %s, "environment": %s,
"inputs": %s, "inputs": %s,
"output": "%s" "output": "%s"
}""" }"""
) ) % (
% ( algorithm_hash,
algorithm_hash, _stringify(parameters),
_stringify(parameters), _stringify(environment),
_stringify(environment), _stringify(input_hashes),
_stringify(input_hashes), output_name,
output_name,
)
) )
return hash(s) return hash(s)
...@@ -131,23 +128,20 @@ def hashAnalyzer( ...@@ -131,23 +128,20 @@ def hashAnalyzer(
# Note: 'analyzer_name' isn't used to compute the hash, but is useful when # Note: 'analyzer_name' isn't used to compute the hash, but is useful when
# an applications want to implement its own hash function # an applications want to implement its own hash function
s = ( s = _compact(
_compact( """{
"""{
"algorithm_name": "%s", "algorithm_name": "%s",
"algorithm": "%s", "algorithm": "%s",
"parameters": %s, "parameters": %s,
"environment": %s, "environment": %s,
"inputs": %s "inputs": %s
}""" }"""
) ) % (
% ( algorithm_name,
algorithm_name, algorithm_hash,
algorithm_hash, _stringify(parameters),
_stringify(parameters), _stringify(environment),
_stringify(environment), _stringify(input_hashes),
_stringify(input_hashes),
)
) )
return hash(s) return hash(s)
......
...@@ -277,8 +277,7 @@ class Plotter(object): ...@@ -277,8 +277,7 @@ class Plotter(object):
_check_language_consistence = algorithm.Algorithm._check_language_consistence _check_language_consistence = algorithm.Algorithm._check_language_consistence
def _validate_dataformat(self, dataformat_cache): def _validate_dataformat(self, dataformat_cache):
"""Makes sure we can load the requested format """Makes sure we can load the requested format"""
"""
name = self.data["dataformat"] name = self.data["dataformat"]
...@@ -304,8 +303,7 @@ class Plotter(object): ...@@ -304,8 +303,7 @@ class Plotter(object):
@property @property
def name(self): def name(self):
"""The name of this object """The name of this object"""
"""
return self._name or "__unnamed_plotter__" return self._name or "__unnamed_plotter__"
@name.setter @name.setter
......
...@@ -7,7 +7,12 @@ from beat.backend.python.database import View ...@@ -7,7 +7,12 @@ from beat.backend.python.database import View
class FooView(View): class FooView(View):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -67,25 +67,25 @@ def load_schema(schema_name, version=1): ...@@ -67,25 +67,25 @@ def load_schema(schema_name, version=1):
"""Returns a JSON validator for the schema given the relative name """Returns a JSON validator for the schema given the relative name
Parameters: Parameters:
schema_name (str): the name of the schema to load. This value corresponds schema_name (str): the name of the schema to load. This value corresponds
to the filename inside our schema directory (where this file is located) to the filename inside our schema directory (where this file is located)
and should *exclude* the extension ``.json``. and should *exclude* the extension ``.json``.
version (int): the version of the schema to use. version (int): the version of the schema to use.
Returns: Returns:
jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator. jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
Raises: Raises:
jsonschema.SchemaError: If there is an error on the schema. jsonschema.SchemaError: If there is an error on the schema.
""" """
fname = pkg_resources.resource_filename( fname = pkg_resources.resource_filename(
__name__, os.path.join(schema_name, "%d.json" % version) __name__, os.path.join(schema_name, "%d.json" % version)
...@@ -109,54 +109,54 @@ def load_schema(schema_name, version=1): ...@@ -109,54 +109,54 @@ def load_schema(schema_name, version=1):
def validate(schema_name, data): def validate(schema_name, data):
"""Validates the input data using the schema """Validates the input data using the schema
This function handles schema versionning in the context of BEAT transparently This function handles schema versionning in the context of BEAT transparently
by first peeking the schema version required by the JSON data and then by first peeking the schema version required by the JSON data and then
validating the JSON data against the proper schema version for the respective validating the JSON data against the proper schema version for the respective
type. type.
Example: Example:
.. code-block:: python .. code-block:: python
try: try:
cleaned_data, error_list = validate('toolchain', '/to/my/file.json') cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
print(e) print(e)
Parameters: Parameters:
schema_name (str): The relative path to the schema to use for validation. schema_name (str): The relative path to the schema to use for validation.
For example, to validate a toolchain, use ``'toolchain'``. For example, to validate a toolchain, use ``'toolchain'``.
data (object, str, file): The piece of data to validate. The input can be a data (object, str, file): The piece of data to validate. The input can be a
valid python object that represents a JSON structure, a file, from which valid python object that represents a JSON structure, a file, from which
the JSON contents will be read out or a string. the JSON contents will be read out or a string.
If ``data`` is a string and represents a valid filesystem path, the If ``data`` is a string and represents a valid filesystem path, the
relevant file will be opened and read as with relevant file will be opened and read as with
:py:func:`json.load``. Otherwise, it will be considered to be :py:func:`json.load``. Otherwise, it will be considered to be
string containing a valid JSON structure that will be loaded as with string containing a valid JSON structure that will be loaded as with
:py:func:`json.loads`. :py:func:`json.loads`.
Note that if the file is opened and read internally using Note that if the file is opened and read internally using
:py:func:`json.load`, exceptions may be thrown by that subsystem, :py:func:`json.load`, exceptions may be thrown by that subsystem,
concerning the file structure. Consult the manual page for concerning the file structure. Consult the manual page for
:py:mod:`simplejson` for details. :py:mod:`simplejson` for details.
Returns: Returns:
A tuple with two elements: the cleaned JSON data structure, after A tuple with two elements: the cleaned JSON data structure, after
processing and a list of errors found by ``jsonschema``. If no errors processing and a list of errors found by ``jsonschema``. If no errors
occur, then returns an empty list for the second element of the tuple. occur, then returns an empty list for the second element of the tuple.
Raises: Raises:
jsonschema.SchemaError: If there is an error on the schema. jsonschema.SchemaError: If there is an error on the schema.
""" """
try: try:
data = maybe_load_json(data) data = maybe_load_json(data)
......
...@@ -53,9 +53,11 @@ Options: ...@@ -53,9 +53,11 @@ Options:
import copy import copy
import os import os
import sys import sys
from collections import OrderedDict from collections import OrderedDict
import simplejson as json import simplejson as json
from docopt import docopt from docopt import docopt
from ..database import Database from ..database import Database
......
...@@ -72,6 +72,7 @@ except ImportError: ...@@ -72,6 +72,7 @@ except ImportError:
import queue as Queue import queue as Queue
import tempfile import tempfile
from socket import gethostname from socket import gethostname
from docopt import docopt from docopt import docopt
...@@ -84,7 +85,7 @@ from ..version import __version__ ...@@ -84,7 +85,7 @@ from ..version import __version__
from ..worker import WorkerController from ..worker import WorkerController
stop = False stop = False
logger = None logger = logging.getLogger(__name__)
# ---------------------------------------------------------- # ----------------------------------------------------------
...@@ -250,9 +251,6 @@ def main(user_input=None): ...@@ -250,9 +251,6 @@ def main(user_input=None):
beat_core_logger.setLevel(logging.WARNING) beat_core_logger.setLevel(logging.WARNING)
beat_backend_logger.setLevel(logging.WARNING) beat_backend_logger.setLevel(logging.WARNING)
global logger
logger = logging.getLogger(__name__)
# Check the prefix path # Check the prefix path
prefix = args["--prefix"] if args["--prefix"] is not None else "." prefix = args["--prefix"] if args["--prefix"] is not None else "."
if not os.path.exists(prefix): if not os.path.exists(prefix):
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):
def setup( def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None, self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
): ):
"""Initializes the database""" """Initializes the database"""
......
...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView ...@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView): class View(BaseView):