Commit 7ab0f93a authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'pyproject_toml' into 'master'

Add pyproject.toml

See merge request !137
parents f84d0710 0a0ccef1
Pipeline #49652 failed with stages
in 174 minutes and 51 seconds
......@@ -6,3 +6,5 @@ use_parentheses=true
ensure_newline_before_comments=true
line_length=88
force_single_line=true
order_by_type=true
lines_between_types=1
......@@ -2,17 +2,17 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/timothycrosley/isort
rev: 5.0.4
rev: 5.8.0
hooks:
- id: isort
files: .*.py
- repo: https://github.com/psf/black
rev: 19.10b0
rev: 20.8b1
hooks:
- id: black
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.1.0
rev: v3.4.0
hooks:
- id: check-ast
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......@@ -26,11 +26,11 @@ repos:
- id: check-yaml
exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/flake8
rev: 3.8.3
rev: 3.9.0
hooks:
- id: flake8
- repo: https://github.com/PyCQA/bandit
rev: 1.6.2
rev: 1.7.0
hooks:
- id: bandit
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......
......@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm):
self._check_language_consistence()
def _check_endpoint_uniqueness(self):
"""Checks for name clashes accross input/output groups
"""
"""Checks for name clashes accross input/output groups"""
all_input_names = []
for group in self.groups:
......@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm):
self._validate_format(type_name, group_name, name, thisformat)
def _validate_required_dataformats(self, dataformat_cache):
"""Makes sure we can load all requested formats
"""
"""Makes sure we can load all requested formats"""
for group in self.groups:
......
......@@ -65,10 +65,12 @@ import os
import signal
import sys
import tempfile
from socket import gethostname
import simplejson as json
import zmq
from docopt import docopt
from ..bcpapi import BCP
......
......@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker
import logging
import signal
import time
from binascii import hexlify
import zmq
......
......@@ -73,8 +73,7 @@ class BeatComputationClient(object):
logger.info("I: connecting to broker at %s...", self.broker)
def send(self, service, request):
"""Send request to broker
"""
"""Send request to broker"""
if not isinstance(request, list):
request = [request]
......
......@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__)
class BeatComputationProcessor(object):
"""BEAT Computation Protocol Processor API, Python version
"""
"""BEAT Computation Protocol Processor API, Python version"""
def __init__(self, poller, address, verbose=False):
self.verbose = verbose
......
......@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__)
class BeatComputationWorker(object):
"""BEAT Computation Protocol Worker API, Python version
"""
"""BEAT Computation Protocol Worker API, Python version"""
HEARTBEAT_LIVENESS = 3 # 3-5 is reasonable
......
......@@ -51,6 +51,7 @@ import time
import docker
import simplejson as json
from packaging import version
from beat.core import stats
......@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__)
class Host(object):
"""An object of this class can connect to the docker host and resolve stuff
"""
"""An object of this class can connect to the docker host and resolve stuff"""
images_cache = {}
......@@ -807,19 +807,17 @@ class Container:
self._temporary_filesystems[path] = size
def set_name(self, name):
""" Set the name to be used by the container in place of the docker
"""Set the name to be used by the container in place of the docker
auto generated one.
"""
self._name = name
def set_workdir(self, workdir):
""" Set the work folder to be used by the container
"""
"""Set the work folder to be used by the container"""
self._workdir = workdir
def set_entrypoint(self, entrypoint):
""" Set the entry point to be used by the container
"""
"""Set the entry point to be used by the container"""
self._entrypoint = entrypoint
def add_volume(self, path, mount_path, read_only=True):
......@@ -952,8 +950,7 @@ class Container:
@property
def environment_variables(self):
"""Returns the environment variables to set on this container.
"""
"""Returns the environment variables to set on this container."""
environment_variables = []
for k, v in self._environment_variables.items():
......
......@@ -302,8 +302,7 @@ class BaseExecutor(object):
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Closes all sinks and disconnects inputs and outputs
"""
"""Closes all sinks and disconnects inputs and outputs"""
for sink in self.data_sinks:
# we save the output only if no valid error has been thrown
......
......@@ -44,6 +44,7 @@ Execution utilities
import logging
import os
import shutil
from collections import namedtuple
import requests
......@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor):
databases_infos = {}
for db_name, db_object, in self.databases.items():
for (
db_name,
db_object,
) in self.databases.items():
json_path = os.path.join(root_folder, db_name + ".json")
with open(json_path, "r") as f:
......
......@@ -86,8 +86,7 @@ def hashBlockOutput(
# Note: 'block_name' and 'algorithm_name' aren't used to compute the hash,
# but are useful when an application wants to implement its own hash
# function
s = (
_compact(
s = _compact(
"""{
"algorithm": "%s",
"parameters": %s,
......@@ -95,15 +94,13 @@ def hashBlockOutput(
"inputs": %s,
"output": "%s"
}"""
)
% (
) % (
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
output_name,
)
)
return hash(s)
......@@ -131,8 +128,7 @@ def hashAnalyzer(
# Note: 'analyzer_name' isn't used to compute the hash, but is useful when
# an applications want to implement its own hash function
s = (
_compact(
s = _compact(
"""{
"algorithm_name": "%s",
"algorithm": "%s",
......@@ -140,15 +136,13 @@ def hashAnalyzer(
"environment": %s,
"inputs": %s
}"""
)
% (
) % (
algorithm_name,
algorithm_hash,
_stringify(parameters),
_stringify(environment),
_stringify(input_hashes),
)
)
return hash(s)
......
......@@ -277,8 +277,7 @@ class Plotter(object):
_check_language_consistence = algorithm.Algorithm._check_language_consistence
def _validate_dataformat(self, dataformat_cache):
"""Makes sure we can load the requested format
"""
"""Makes sure we can load the requested format"""
name = self.data["dataformat"]
......@@ -304,8 +303,7 @@ class Plotter(object):
@property
def name(self):
"""The name of this object
"""
"""The name of this object"""
return self._name or "__unnamed_plotter__"
@name.setter
......
......@@ -7,7 +7,12 @@ from beat.backend.python.database import View
class FooView(View):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -53,9 +53,11 @@ Options:
import copy
import os
import sys
from collections import OrderedDict
import simplejson as json
from docopt import docopt
from ..database import Database
......
......@@ -72,6 +72,7 @@ except ImportError:
import queue as Queue
import tempfile
from socket import gethostname
from docopt import docopt
......@@ -84,7 +85,7 @@ from ..version import __version__
from ..worker import WorkerController
stop = False
logger = None
logger = logging.getLogger(__name__)
# ----------------------------------------------------------
......@@ -250,9 +251,6 @@ def main(user_input=None):
beat_core_logger.setLevel(logging.WARNING)
beat_backend_logger.setLevel(logging.WARNING)
global logger
logger = logging.getLogger(__name__)
# Check the prefix path
prefix = args["--prefix"] if args["--prefix"] is not None else "."
if not os.path.exists(prefix):
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
self,
root_folder,
outputs,
parameters,
start_index=None,
end_index=None,
):
"""Initializes the database"""
......
......@@ -37,12 +37,14 @@
# Tests for experiment execution
import multiprocessing
import multiprocessing.queues
import os
import queue
import unittest
import simplejson as json
import zmq
from flaky import flaky
from ..bcp import broker
......@@ -134,24 +136,37 @@ DEFAULT_MAX_ITERATION_COUNT = 30
class ZMQBrokerProcess(multiprocessing.Process):
def __init__(self, port, verbose, callbacks=None):
def __init__(self, port, verbose, queue=None):
super(ZMQBrokerProcess, self).__init__()
self.port = port
self.verbose = verbose
self.callbacks = callbacks
self.queue = queue
def run(self):
return broker.run(self.port, verbose=self.verbose, callbacks=self.callbacks)
callbacks = None
if self.queue:
callbacks = self.queue.callbacks()
return broker.run(self.port, verbose=self.verbose, callbacks=callbacks)
class ZMQWorkerProcess(multiprocessing.Process):
def __init__(
self, address, name, verbose, use_docker=False, docker_images_cache=None
self,
address,
name,
verbose,
prefix,
tmp_prefix,
use_docker=False,
docker_images_cache=None,
):
super(ZMQWorkerProcess, self).__init__()
self.broker_address = address
self.service_name = name
self.verbose = verbose
self.prefix = prefix
self.tmp_prefix = tmp_prefix
self.use_docker = use_docker
self.docker_images_cache = None
......@@ -160,8 +175,8 @@ class ZMQWorkerProcess(multiprocessing.Process):
self.broker_address,
service_name=self.service_name,
verbose=self.verbose,
prefix=prefix,
cache=tmp_prefix,
prefix=self.prefix,
cache=self.tmp_prefix,
use_docker=self.use_docker,
docker_images_cache=self.docker_images_cache,
)
......@@ -183,15 +198,27 @@ class ExecutionTestCase(unittest.TestCase):
view.index(os.path.join(tmp_prefix, input_cfg["path"]))
class TestBroker(unittest.TestCase):
class CallbackedQueue(multiprocessing.queues.Queue):
def __init__(self, *args, **kwargs):
from multiprocessing.context import BaseContext
ctx = BaseContext()
ctx._name = "callbacks"
super().__init__(*args, **kwargs, ctx=ctx)
def __on_ready(self, name):
self.queue.put("ready")
self.put("ready")
def __on_gone(self, name):
self.queue.put("gone")
self.put("gone")
def callbacks(self):
return (self.__on_ready, self.__on_gone)
class TestBroker(unittest.TestCase):
def setUp(self):
self.queue = multiprocessing.Queue()
self.queue = CallbackedQueue()
def test_callback(self):
worker_name = b"test_worker"
......@@ -199,12 +226,12 @@ class TestBroker(unittest.TestCase):
port = find_free_port()
broker_address = "tcp://localhost:{}".format(port)
broker_p = ZMQBrokerProcess(
port, VERBOSE_BCP_LOGGING, (self.__on_ready, self.__on_gone)
)
broker_p = ZMQBrokerProcess(port, VERBOSE_BCP_LOGGING, self.queue)
broker_p.start()
worker = ZMQWorkerProcess(broker_address, worker_name, VERBOSE_BCP_LOGGING)
worker = ZMQWorkerProcess(
broker_address, worker_name, VERBOSE_BCP_LOGGING, prefix, tmp_prefix
)
worker.start()
worker.join(2) # Give the worker enough time to announce itself
worker.terminate()
......@@ -248,6 +275,8 @@ class TestBCP(ExecutionTestCase):
broker_address,
self.worker_name,
VERBOSE_BCP_LOGGING,
prefix,
tmp_prefix,
self.use_docker,
self.docker_images_cache,
)
......
......@@ -39,6 +39,7 @@ import os
import tempfile
import time
import unittest
from tempfile import TemporaryDirectory
import pkg_resources
......
......@@ -306,7 +306,8 @@ class TestDockerExecution(BaseExecutionMixIn):
def test_loop_mix_db_env_error(self):
with nose.tools.assert_raises(RuntimeError) as context:
self.execute(
"errors/user/loop/1/loop_mix_db_env", [None],
"errors/user/loop/1/loop_mix_db_env",
[None],
)
nose.tools.assert_true(
......@@ -317,7 +318,8 @@ class TestDockerExecution(BaseExecutionMixIn):
def test_loop_two_db_env_error(self):
with nose.tools.assert_raises(RuntimeError) as context:
self.execute(
"errors/user/loop/1/loop_two_db_environments", [None],
"errors/user/loop/1/loop_two_db_environments",
[None],
)
nose.tools.assert_true(
......@@ -328,7 +330,8 @@ class TestDockerExecution(BaseExecutionMixIn):
def test_single_not_existing_db_env_error(self):
with nose.tools.assert_raises(RuntimeError) as context:
self.execute(
"errors/user/single/1/not_existing_db_env", [None],
"errors/user/single/1/not_existing_db_env",
[None],
)
nose.tools.assert_true(
......
......@@ -40,6 +40,7 @@ import glob
import logging
import os
import subprocess as sp # nosec
from shutil import rmtree
import nose.tools
......@@ -95,8 +96,7 @@ def clear_conda_environment(environment_prefix):
class BaseExecutionMixIn(object):
def check_output(self, prefix, path):
"""Checks if a given output exists, together with its indexes and checksums
"""
"""Checks if a given output exists, together with its indexes and checksums"""
finalpath = os.path.join(prefix, path)
datafiles = glob.glob(finalpath + "*.data")
......
......@@ -40,10 +40,12 @@ import multiprocessing
import os
import queue
import unittest
from time import sleep
from time import time
import simplejson as json
from ddt import ddt
from ddt import idata
......@@ -147,10 +149,11 @@ def prepare_database(db_name):
class ControllerProcess(multiprocessing.Process):
def __init__(self, queue):
def __init__(self, queue, port):
super(ControllerProcess, self).__init__()
self.queue = queue
self.port = port
def run(self):
self.queue.put("STARTED")
......@@ -163,7 +166,7 @@ class ControllerProcess(multiprocessing.Process):
self.controller = WorkerController(
"127.0.0.1",
port=PORT,
port=self.port,
callbacks=dict(onWorkerReady=onWorkerReady, onWorkerGone=onWorkerGone),
)
......@@ -338,7 +341,7 @@ class TestConnection(TestWorkerBase):
self.wait_for_worker_connection(WORKER1)
def test_scheduler_shutdown(self):
controller = ControllerProcess(multiprocessing.Queue())
controller = ControllerProcess(multiprocessing.Queue(), PORT)
controller.start()
message = controller.queue.get()
......@@ -353,7 +356,7 @@ class TestConnection(TestWorkerBase):
sleep(1)
controller = ControllerProcess(multiprocessing.Queue())
controller = ControllerProcess(multiprocessing.Queue(), PORT)
controller.start()
message = controller.queue.get()
......
......@@ -135,7 +135,7 @@ def find_free_port_in_range(min_port, max_port):
def id_generator(
size=6, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits
):
""" Simple id generator based on
"""Simple id generator based on
https://stackoverflow.com/a/2257449/5843716
"""
return "".join(random.choice(chars) for _ in range(size)) # nosec
......
......@@ -39,6 +39,7 @@ import time
import pkg_resources
import sphinx_rtd_theme
from bob.extension.utils import link_documentation
from bob.extension.utils import load_requirements
......
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment