Commit 057a240d authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'pre_commit_cleanup' into 'master'

Pre commit cleanup

See merge request !126
parents 8ea89087 4c771e17
Pipeline #41802 passed with stages
in 39 minutes and 6 seconds
[flake8] [flake8]
max-line-length = 80 max-line-length = 88
select = B,C,E,F,W,T4,B9,B950 select = B,C,E,F,W,T4,B9,B950
ignore = E501, W503, E203 ignore = E501, W503, E203
[settings]
multi_line_output=3
include_trailing_comma=true
force_grid_wrap=0
use_parentheses=true
ensure_newline_before_comments=true
line_length=88
force_single_line=true
# See https://pre-commit.com for more information # See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks # See https://pre-commit.com/hooks.html for more hooks
repos: repos:
- repo: https://github.com/ambv/black - repo: https://github.com/timothycrosley/isort
rev: stable rev: 5.0.4
hooks: hooks:
- id: black - id: isort
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py files: .*.py
- repo: https://github.com/psf/black
rev: 19.10b0
hooks:
- id: black
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.0.0 rev: v3.1.0
hooks: hooks:
- id: check-ast
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- id: check-case-conflict
- id: trailing-whitespace - id: trailing-whitespace
- id: end-of-file-fixer - id: end-of-file-fixer
- id: debug-statements - id: debug-statements
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- id: check-added-large-files - id: check-added-large-files
- id: check-docstring-first - id: check-docstring-first
- id: flake8
- id: check-yaml - id: check-yaml
exclude: conda/meta.yaml exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/flake8
rev: 3.8.3
hooks:
- id: flake8
- repo: https://github.com/PyCQA/bandit - repo: https://github.com/PyCQA/bandit
rev: 'master' # Update me! rev: 1.6.2
hooks: hooks:
- id: bandit - id: bandit
exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
......
...@@ -33,8 +33,8 @@ ...@@ -33,8 +33,8 @@
# # # #
################################################################################### ###################################################################################
# see https://docs.python.org/3/library/pkgutil.html # see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path from pkgutil import extend_path
__path__ = extend_path(__path__, __name__) __path__ = extend_path(__path__, __name__)
...@@ -45,22 +45,21 @@ Forward importing from :py:mod:`beat.backend.python.algorithm` ...@@ -45,22 +45,21 @@ Forward importing from :py:mod:`beat.backend.python.algorithm`
:py:class:`beat.backend.python.algorithm.Storage` :py:class:`beat.backend.python.algorithm.Storage`
:py:class:`beat.backend.python.algorithm.Runner` :py:class:`beat.backend.python.algorithm.Runner`
""" """
import os import os
import six
import numpy import numpy
import pkg_resources import pkg_resources
import simplejson as json import simplejson as json
import six
from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
from beat.backend.python.algorithm import Runner # noqa
from beat.backend.python.algorithm import Storage
from . import dataformat from . import dataformat
from . import library from . import library
from . import schema
from . import prototypes from . import prototypes
from . import schema
from beat.backend.python.algorithm import Storage
from beat.backend.python.algorithm import Runner # noqa
from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
def load_algorithm_prototype(prefix): def load_algorithm_prototype(prefix):
......
...@@ -49,15 +49,14 @@ Options: ...@@ -49,15 +49,14 @@ Options:
-V, --version Show version -V, --version Show version
-v, --verbose Increases the output verbosity level -v, --verbose Increases the output verbosity level
""" """
import os import os
import sys import sys
from docopt import docopt from docopt import docopt
from ..bcpapi.broker import BeatComputationBroker from ..bcpapi.broker import BeatComputationBroker
from ..version import __version__
from ..utils import setup_logging from ..utils import setup_logging
from ..version import __version__
def run(port=5555, verbose=1, callbacks=None): def run(port=5555, verbose=1, callbacks=None):
......
...@@ -61,28 +61,25 @@ Options: ...@@ -61,28 +61,25 @@ Options:
--cache-mount-point=<cache_mount_point> NFS mount point to use for cache setup --cache-mount-point=<cache_mount_point> NFS mount point to use for cache setup
""" """
import os import os
import sys
import signal import signal
import sys
import tempfile import tempfile
import simplejson as json from socket import gethostname
import simplejson as json
import zmq import zmq
from socket import gethostname
from docopt import docopt from docopt import docopt
from ..bcpapi import BCP from ..bcpapi import BCP
from ..bcpapi.worker import BeatComputationWorker
from ..bcpapi.processor import BeatComputationProcessor
from ..bcpapi.execution import ExecutionProcess from ..bcpapi.execution import ExecutionProcess
from ..bcpapi.processor import BeatComputationProcessor
from ..bcpapi.worker import BeatComputationWorker
from ..dock import Host from ..dock import Host
from ..utils import find_free_port from ..utils import find_free_port
from ..utils import setup_logging from ..utils import setup_logging
from ..version import __version__ from ..version import __version__
logger = None logger = None
......
...@@ -37,20 +37,16 @@ ...@@ -37,20 +37,16 @@
Inspired by the Majordomo Protocol Broker Inspired by the Majordomo Protocol Broker
""" """
import logging import logging
import time
import signal import signal
import time
from binascii import hexlify from binascii import hexlify
import zmq import zmq
# local
from . import BCP from . import BCP
from .zhelpers import dump from .zhelpers import dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -40,17 +40,16 @@ management ...@@ -40,17 +40,16 @@ management
Execution utilities Execution utilities
""" """
import logging import logging
import multiprocessing import multiprocessing
import signal import signal
import zmq
import simplejson as json import simplejson as json
import zmq
from ..dock import Host from ..dock import Host
from ..execution.local import LocalExecutor
from ..execution.docker import DockerExecutor from ..execution.docker import DockerExecutor
from ..execution.local import LocalExecutor
from . import BCP from . import BCP
......
...@@ -34,8 +34,8 @@ ...@@ -34,8 +34,8 @@
"""BEAT Computation worker""" """BEAT Computation worker"""
import logging import logging
import zmq import zmq
from .zhelpers import dump from .zhelpers import dump
......
...@@ -34,14 +34,12 @@ ...@@ -34,14 +34,12 @@
"""BEAT Computation worker""" """BEAT Computation worker"""
import logging import logging
import time import time
import zmq import zmq
# BEAT Computation protocol constants:
from . import BCP from . import BCP
from .zhelpers import dump from .zhelpers import dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -58,8 +56,8 @@ class BeatComputationWorker(object): ...@@ -58,8 +56,8 @@ class BeatComputationWorker(object):
service = service.encode("utf-8") service = service.encode("utf-8")
self.heartbeat_at = ( self.heartbeat_at = (
0 0 # When to send HEARTBEAT (relative to time.time(), so in seconds)
) # When to send HEARTBEAT (relative to time.time(), so in seconds) )
self.liveness = 0 # How many attempts left self.liveness = 0 # How many attempts left
self.heartbeat = 2500 # Heartbeat delay, msecs self.heartbeat = 2500 # Heartbeat delay, msecs
self.reconnect = 2500 # Reconnect delay, msecs self.reconnect = 2500 # Reconnect delay, msecs
......
...@@ -38,9 +38,9 @@ Helper module for common zmq task ...@@ -38,9 +38,9 @@ Helper module for common zmq task
Based on Majordomo protocol zhelpers. Based on Majordomo protocol zhelpers.
""" """
import binascii import binascii
import logging import logging
import zmq import zmq
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -53,16 +53,15 @@ Forward importing from :py:mod:`beat.backend.python.data`: ...@@ -53,16 +53,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
:py:func:`beat.backend.python.data.load_data_index_db` :py:func:`beat.backend.python.data.load_data_index_db`
:py:func:`beat.backend.python.data.foundSplitRanges` :py:func:`beat.backend.python.data.foundSplitRanges`
""" """
from beat.backend.python.data import CachedDataSink # noqa
from beat.backend.python.data import mixDataIndices # noqa
from beat.backend.python.data import getAllFilenames # noqa
from beat.backend.python.data import DataSource # noqa
from beat.backend.python.data import CachedDataSource # noqa from beat.backend.python.data import CachedDataSource # noqa
from beat.backend.python.data import DatabaseOutputDataSource # noqa from beat.backend.python.data import DatabaseOutputDataSource # noqa
from beat.backend.python.data import RemoteDataSource # noqa
from beat.backend.python.data import DataSink # noqa from beat.backend.python.data import DataSink # noqa
from beat.backend.python.data import CachedDataSink # noqa from beat.backend.python.data import DataSource # noqa
from beat.backend.python.data import RemoteDataSource # noqa
from beat.backend.python.data import StdoutDataSink # noqa from beat.backend.python.data import StdoutDataSink # noqa
from beat.backend.python.data import foundSplitRanges # noqa
from beat.backend.python.data import getAllFilenames # noqa
from beat.backend.python.data import load_data_index # noqa from beat.backend.python.data import load_data_index # noqa
from beat.backend.python.data import load_data_index_db # noqa from beat.backend.python.data import load_data_index_db # noqa
from beat.backend.python.data import foundSplitRanges # noqa from beat.backend.python.data import mixDataIndices # noqa
...@@ -44,8 +44,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders` ...@@ -44,8 +44,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
:py:class:`beat.backend.python.data_loaders.DataLoader` :py:class:`beat.backend.python.data_loaders.DataLoader`
:py:class:`beat.backend.python.data_loaders.DataView` :py:class:`beat.backend.python.data_loaders.DataView`
""" """
from beat.backend.python.data_loaders import DataLoaderList # noqa
from beat.backend.python.data_loaders import DataLoader # noqa from beat.backend.python.data_loaders import DataLoader # noqa
from beat.backend.python.data_loaders import DataLoaderList # noqa
from beat.backend.python.data_loaders import DataView # noqa from beat.backend.python.data_loaders import DataView # noqa
...@@ -44,22 +44,19 @@ Validation of databases ...@@ -44,22 +44,19 @@ Validation of databases
Forward importing from :py:mod:`beat.backend.python.database`: Forward importing from :py:mod:`beat.backend.python.database`:
:py:class:`beat.backend.python.database.Storage` :py:class:`beat.backend.python.database.Storage`
""" """
import os import os
import six import six
from . import schema from beat.backend.python.database import Database as BackendDatabase
from beat.backend.python.database import Storage
from beat.backend.python.protocoltemplate import Storage as PTStorage
from . import prototypes
from . import schema
from .dataformat import DataFormat from .dataformat import DataFormat
from .protocoltemplate import ProtocolTemplate from .protocoltemplate import ProtocolTemplate
from . import prototypes
from beat.backend.python.database import Storage
from beat.backend.python.database import Database as BackendDatabase
from beat.backend.python.protocoltemplate import Storage as PTStorage
def get_first_procotol_template(prefix): def get_first_procotol_template(prefix):
pt_root_folder = os.path.join(prefix, PTStorage.asset_folder) pt_root_folder = os.path.join(prefix, PTStorage.asset_folder)
......
...@@ -44,17 +44,17 @@ Validation and parsing for dataformats ...@@ -44,17 +44,17 @@ Validation and parsing for dataformats
Forward importing from :py:mod:`beat.backend.python.dataformat`: Forward importing from :py:mod:`beat.backend.python.dataformat`:
:py:class:`beat.backend.python.dataformat.Storage` :py:class:`beat.backend.python.dataformat.Storage`
""" """
import copy import copy
import six import six
from . import schema from beat.backend.python.dataformat import DataFormat as BackendDataFormat
from beat.backend.python.dataformat import Storage # noqa
from . import prototypes from . import prototypes
from . import schema
from . import utils from . import utils
from beat.backend.python.dataformat import Storage # noqa
from beat.backend.python.dataformat import DataFormat as BackendDataFormat
class DataFormat(BackendDataFormat): class DataFormat(BackendDataFormat):
"""Data formats define the chunks of data that circulate between blocks. """Data formats define the chunks of data that circulate between blocks.
......
...@@ -41,18 +41,16 @@ dock ...@@ -41,18 +41,16 @@ dock
Docker helper classes Docker helper classes
""" """
import ast import ast
import logging
import os import os
import simplejson as json
import socket import socket
import subprocess as sp # nosec
import tempfile import tempfile
import time import time
import docker
import subprocess as sp # nosec
import logging
import docker
import simplejson as json
from packaging import version from packaging import version
from beat.core import stats from beat.core import stats
......
...@@ -41,10 +41,8 @@ environment ...@@ -41,10 +41,8 @@ environment
Helper functions related to environment management Helper functions related to environment management
""" """
import re
import logging import logging
import re
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -41,21 +41,19 @@ base ...@@ -41,21 +41,19 @@ base
Execution utilities Execution utilities
""" """
import os
import glob
import collections import collections
import glob
import logging import logging
import simplejson as json import os
from .. import schema import simplejson as json
from .. import database
from .. import algorithm
from .. import stats
from beat.backend.python.helpers import convert_experiment_configuration_to_container from beat.backend.python.helpers import convert_experiment_configuration_to_container
from .. import algorithm
from .. import database
from .. import schema
from .. import stats
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -41,24 +41,21 @@ docker ...@@ -41,24 +41,21 @@ docker
Execution utilities Execution utilities
""" """
import logging
import os import os
import shutil import shutil
import logging from collections import namedtuple
import requests import requests
import simplejson as json import simplejson as json
from collections import namedtuple
from beat.backend.python.execution import MessageHandler
from beat.backend.python.data import getAllFilenames from beat.backend.python.data import getAllFilenames
from beat.backend.python.execution import MessageHandler
from .. import stats from .. import stats
from .. import utils from .. import utils
from .remote import RemoteExecutor from .remote import RemoteExecutor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -41,20 +41,19 @@ local ...@@ -41,20 +41,19 @@ local
Execution utilities Execution utilities
""" """
import logging
import os import os
import shutil
import sys import sys
import tempfile import tempfile
import shutil
import zmq
import time import time
import logging
import zmq
from beat.backend.python.execution import AlgorithmExecutor from beat.backend.python.execution import AlgorithmExecutor
from beat.backend.python.execution import MessageHandler
from beat.backend.python.execution import LoopExecutor from beat.backend.python.execution import LoopExecutor
from beat.backend.python.execution import LoopMessageHandler from beat.backend.python.execution import LoopMessageHandler
from beat.backend.python.execution import MessageHandler
from .base import BaseExecutor from .base import BaseExecutor
......
...@@ -41,12 +41,11 @@ remote ...@@ -41,12 +41,11 @@ remote
Execution utilities Execution utilities
""" """
from .base import BaseExecutor
from beat.backend.python.helpers import create_inputs_from_configuration from beat.backend.python.helpers import create_inputs_from_configuration
from beat.backend.python.helpers import create_outputs_from_configuration from beat.backend.python.helpers import create_outputs_from_configuration
from .base import BaseExecutor
class RemoteExecutor(BaseExecutor): class RemoteExecutor(BaseExecutor):
"""Base class for Executors that communicate with a message handler """Base class for Executors that communicate with a message handler
......
...@@ -40,22 +40,19 @@ subprocess ...@@ -40,22 +40,19 @@ subprocess
Execution utilities Execution utilities
""" """
import logging
import os import os
import shutil import shutil
import logging
import sys
import subprocess as sp # nosec import subprocess as sp # nosec
import sys
import tempfile import tempfile
from beat.backend.python.execution import MessageHandler from beat.backend.python.execution import MessageHandler
from .. import stats from .. import stats
from .. import utils from .. import utils
from .remote import RemoteExecutor from .remote import RemoteExecutor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -41,19 +41,18 @@ experiment ...@@ -41,19 +41,18 @@ experiment
Validation for experiments Validation for experiments
""" """
import os