Merge branch 'pre_commit_cleanup' into 'master'

Pre commit cleanup

See merge request !126

.flake8

 [flake8]
-max-line-length = 80
+max-line-length = 88
 select = B,C,E,F,W,T4,B9,B950
 ignore = E501, W503, E203

.isort.cfg

+[settings]
+multi_line_output=3
+include_trailing_comma=true
+force_grid_wrap=0
+use_parentheses=true
+ensure_newline_before_comments=true
+line_length=88
+force_single_line=true

.pre-commit-config.yaml

 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/ambv/black
-    rev: stable
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.0.4
     hooks:
-    - id: black
-      exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
+      - id: isort
+        files: .*.py
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+        exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.0.0
+    rev: v3.1.0
     hooks:
+      - id: check-ast
+        exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
+      - id: check-case-conflict
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
         exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
       - id: check-added-large-files
       - id: check-docstring-first
-      - id: flake8
       - id: check-yaml
         exclude: conda/meta.yaml
+  - repo: https://github.com/PyCQA/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
   - repo: https://github.com/PyCQA/bandit
-    rev: 'master' # Update me!
+    rev: 1.6.2
     hooks:
       - id: bandit
         exclude: beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py

beat/__init__.py

@@ -33,8 +33,8 @@
 #                                                                                 #
 ###################################################################################
 
-
 # see https://docs.python.org/3/library/pkgutil.html
+
 from pkgutil import extend_path
 
 __path__ = extend_path(__path__, __name__)

beat/core/algorithm.py

@@ -45,22 +45,21 @@ Forward importing from :py:mod:`beat.backend.python.algorithm`
 :py:class:`beat.backend.python.algorithm.Storage`
 :py:class:`beat.backend.python.algorithm.Runner`
 """
-
-
 import os
-import six
+
 import numpy
 import pkg_resources
 import simplejson as json
+import six
+
+from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
+from beat.backend.python.algorithm import Runner  # noqa
+from beat.backend.python.algorithm import Storage
 
 from . import dataformat
 from . import library
-from . import schema
 from . import prototypes
-
-from beat.backend.python.algorithm import Storage
-from beat.backend.python.algorithm import Runner  # noqa
-from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
+from . import schema
 
 
 def load_algorithm_prototype(prefix):

beat/core/bcp/broker.py

@@ -49,15 +49,14 @@ Options:
   -V, --version              Show version
   -v, --verbose              Increases the output verbosity level
 """
-
 import os
 import sys
 
 from docopt import docopt
 
 from ..bcpapi.broker import BeatComputationBroker
-from ..version import __version__
 from ..utils import setup_logging
+from ..version import __version__
 
 
 def run(port=5555, verbose=1, callbacks=None):

beat/core/bcp/worker.py

@@ -61,28 +61,25 @@ Options:
   --cache-mount-point=<cache_mount_point> NFS mount point to use for cache setup
 
 """
-
 import os
-import sys
 import signal
+import sys
 import tempfile
-import simplejson as json
+from socket import gethostname
 
+import simplejson as json
 import zmq
-
-from socket import gethostname
 from docopt import docopt
 
 from ..bcpapi import BCP
-from ..bcpapi.worker import BeatComputationWorker
-from ..bcpapi.processor import BeatComputationProcessor
 from ..bcpapi.execution import ExecutionProcess
+from ..bcpapi.processor import BeatComputationProcessor
+from ..bcpapi.worker import BeatComputationWorker
 from ..dock import Host
 from ..utils import find_free_port
 from ..utils import setup_logging
 from ..version import __version__
 
-
 logger = None
 
 

beat/core/bcpapi/broker.py

@@ -37,20 +37,16 @@
 
 Inspired by the Majordomo Protocol Broker
 """
-
 import logging
-import time
 import signal
-
+import time
 from binascii import hexlify
 
 import zmq
 
-# local
 from . import BCP
 from .zhelpers import dump
 
-
 logger = logging.getLogger(__name__)
 
 

beat/core/bcpapi/execution.py

@@ -40,17 +40,16 @@ management
 
 Execution utilities
 """
-
 import logging
 import multiprocessing
 import signal
-import zmq
+
 import simplejson as json
+import zmq
 
 from ..dock import Host
-from ..execution.local import LocalExecutor
 from ..execution.docker import DockerExecutor
-
+from ..execution.local import LocalExecutor
 from . import BCP
 
 

beat/core/bcpapi/processor.py

@@ -34,8 +34,8 @@
 
 
 """BEAT Computation worker"""
-
 import logging
+
 import zmq
 
 from .zhelpers import dump

beat/core/bcpapi/worker.py

@@ -34,14 +34,12 @@
 
 
 """BEAT Computation worker"""
-
 import logging
 import time
+
 import zmq
 
-# BEAT Computation protocol constants:
 from . import BCP
-
 from .zhelpers import dump
 
 logger = logging.getLogger(__name__)
@@ -58,8 +56,8 @@ class BeatComputationWorker(object):
             service = service.encode("utf-8")
 
         self.heartbeat_at = (
-            0
-        )  # When to send HEARTBEAT (relative to time.time(), so in seconds)
+            0  # When to send HEARTBEAT (relative to time.time(), so in seconds)
+        )
         self.liveness = 0  # How many attempts left
         self.heartbeat = 2500  # Heartbeat delay, msecs
         self.reconnect = 2500  # Reconnect delay, msecs

beat/core/bcpapi/zhelpers.py

@@ -38,9 +38,9 @@ Helper module for common zmq task
 
 Based on Majordomo protocol zhelpers.
 """
-
 import binascii
 import logging
+
 import zmq
 
 logger = logging.getLogger(__name__)

beat/core/data.py

@@ -53,16 +53,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
 :py:func:`beat.backend.python.data.load_data_index_db`
 :py:func:`beat.backend.python.data.foundSplitRanges`
 """
-
-from beat.backend.python.data import mixDataIndices  # noqa
-from beat.backend.python.data import getAllFilenames  # noqa
-from beat.backend.python.data import DataSource  # noqa
+from beat.backend.python.data import CachedDataSink  # noqa
 from beat.backend.python.data import CachedDataSource  # noqa
 from beat.backend.python.data import DatabaseOutputDataSource  # noqa
-from beat.backend.python.data import RemoteDataSource  # noqa
 from beat.backend.python.data import DataSink  # noqa
-from beat.backend.python.data import CachedDataSink  # noqa
+from beat.backend.python.data import DataSource  # noqa
+from beat.backend.python.data import RemoteDataSource  # noqa
 from beat.backend.python.data import StdoutDataSink  # noqa
+from beat.backend.python.data import foundSplitRanges  # noqa
+from beat.backend.python.data import getAllFilenames  # noqa
 from beat.backend.python.data import load_data_index  # noqa
 from beat.backend.python.data import load_data_index_db  # noqa
-from beat.backend.python.data import foundSplitRanges  # noqa
+from beat.backend.python.data import mixDataIndices  # noqa

beat/core/data_loaders.py

@@ -44,8 +44,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
 :py:class:`beat.backend.python.data_loaders.DataLoader`
 :py:class:`beat.backend.python.data_loaders.DataView`
 """
-
-
-from beat.backend.python.data_loaders import DataLoaderList  # noqa
 from beat.backend.python.data_loaders import DataLoader  # noqa
+from beat.backend.python.data_loaders import DataLoaderList  # noqa
 from beat.backend.python.data_loaders import DataView  # noqa

beat/core/database.py

@@ -44,22 +44,19 @@ Validation of databases
 Forward importing from :py:mod:`beat.backend.python.database`:
 :py:class:`beat.backend.python.database.Storage`
 """
-
 import os
 
 import six
 
-from . import schema
+from beat.backend.python.database import Database as BackendDatabase
+from beat.backend.python.database import Storage
+from beat.backend.python.protocoltemplate import Storage as PTStorage
 
+from . import prototypes
+from . import schema
 from .dataformat import DataFormat
 from .protocoltemplate import ProtocolTemplate
 
-from . import prototypes
-
-from beat.backend.python.database import Storage
-from beat.backend.python.database import Database as BackendDatabase
-from beat.backend.python.protocoltemplate import Storage as PTStorage
-
 
 def get_first_procotol_template(prefix):
     pt_root_folder = os.path.join(prefix, PTStorage.asset_folder)

beat/core/dataformat.py

@@ -44,17 +44,17 @@ Validation and parsing for dataformats
 Forward importing from :py:mod:`beat.backend.python.dataformat`:
 :py:class:`beat.backend.python.dataformat.Storage`
 """
-
 import copy
+
 import six
 
-from . import schema
+from beat.backend.python.dataformat import DataFormat as BackendDataFormat
+from beat.backend.python.dataformat import Storage  # noqa
+
 from . import prototypes
+from . import schema
 from . import utils
 
-from beat.backend.python.dataformat import Storage  # noqa
-from beat.backend.python.dataformat import DataFormat as BackendDataFormat
-
 
 class DataFormat(BackendDataFormat):
     """Data formats define the chunks of data that circulate between blocks.

beat/core/dock.py

@@ -41,18 +41,16 @@ dock
 
 Docker helper classes
 """
-
-
 import ast
+import logging
 import os
-import simplejson as json
 import socket
+import subprocess as sp  # nosec
 import tempfile
 import time
-import docker
-import subprocess as sp  # nosec
-import logging
 
+import docker
+import simplejson as json
 from packaging import version
 
 from beat.core import stats

beat/core/environments.py

@@ -41,10 +41,8 @@ environment
 
 Helper functions related to environment management
 """
-
-import re
 import logging
-
+import re
 
 logger = logging.getLogger(__name__)
 

beat/core/execution/base.py

@@ -41,21 +41,19 @@ base
 
 Execution utilities
 """
-
-
-import os
-import glob
 import collections
+import glob
 import logging
-import simplejson as json
+import os
 
-from .. import schema
-from .. import database
-from .. import algorithm
-from .. import stats
+import simplejson as json
 
 from beat.backend.python.helpers import convert_experiment_configuration_to_container
 
+from .. import algorithm
+from .. import database
+from .. import schema
+from .. import stats
 
 logger = logging.getLogger(__name__)
 

beat/core/execution/docker.py

@@ -41,24 +41,21 @@ docker
 
 Execution utilities
 """
-
+import logging
 import os
 import shutil
-import logging
+from collections import namedtuple
+
 import requests
 import simplejson as json
 
-from collections import namedtuple
-
-from beat.backend.python.execution import MessageHandler
 from beat.backend.python.data import getAllFilenames
+from beat.backend.python.execution import MessageHandler
 
 from .. import stats
 from .. import utils
-
 from .remote import RemoteExecutor
 
-
 logger = logging.getLogger(__name__)
 
 

beat/core/execution/local.py

@@ -41,20 +41,19 @@ local
 
 Execution utilities
 """
-
-
+import logging
 import os
+import shutil
 import sys
 import tempfile
-import shutil
-import zmq
 import time
-import logging
+
+import zmq
 
 from beat.backend.python.execution import AlgorithmExecutor
-from beat.backend.python.execution import MessageHandler
 from beat.backend.python.execution import LoopExecutor
 from beat.backend.python.execution import LoopMessageHandler
+from beat.backend.python.execution import MessageHandler
 
 from .base import BaseExecutor
 

beat/core/execution/remote.py

@@ -41,12 +41,11 @@ remote
 
 Execution utilities
 """
-
-from .base import BaseExecutor
-
 from beat.backend.python.helpers import create_inputs_from_configuration
 from beat.backend.python.helpers import create_outputs_from_configuration
 
+from .base import BaseExecutor
+
 
 class RemoteExecutor(BaseExecutor):
     """Base class for Executors that communicate with a message handler

beat/core/execution/subprocess.py

@@ -40,22 +40,19 @@ subprocess
 
 Execution utilities
 """
-
+import logging
 import os
 import shutil
-import logging
-import sys
 import subprocess as sp  # nosec
+import sys
 import tempfile
 
 from beat.backend.python.execution import MessageHandler
 
 from .. import stats
 from .. import utils
-
 from .remote import RemoteExecutor
 
-
 logger = logging.getLogger(__name__)
 
 

beat/core/experiment.py

@@ -41,19 +41,18 @@ experiment
 
 Validation for experiments
 """
-
-import os
 import collections
 import itertools
+import os
+
 import simplejson as json
 
-from . import utils
 from . import algorithm
-
-from . import schema
 from . import database
-from . import toolchain
 from . import hash
+from . import schema
+from . import toolchain
+from . import utils
 
 EVALUATOR_PREFIX = "evaluator_"
 PROCESSOR_PREFIX = "processor_"
@@ -84,9 +83,13 @@ class Storage(utils.Storage):
         if len(s) == 4:
             name = os.path.join(s[0], name)
 
-        self.username, self.toolchain_username, self.toolchain, self.version, self.name = name.split(
-            os.sep
-        )
+        (
+            self.username,
+            self.toolchain_username,
+            self.toolchain,
+            self.version,
+            self.name,
+        ) = name.split(os.sep)
 
         self.label = name
         self.toolchain = os.path.join(

beat/core/hash.py

@@ -43,16 +43,14 @@ Various functions for hashing platform contributions and others
 
 Also forward importing from :py:mod:`beat.backend.python.hash`
 """
-
 import collections
 
 import simplejson as json
 
 from beat.backend.python.hash import *  # noqa
+from beat.backend.python.hash import _compact
 from beat.backend.python.hash import _sha256  # noqa
 from beat.backend.python.hash import _stringify
-from beat.backend.python.hash import _compact
-
 
 # ----------------------------------------------------------
 

beat/core/inputs.py

@@ -45,8 +45,6 @@ Forward imported from :py:mod:`beat.backend.python.inputs`:
 :py:class:`beat.backend.python.inputs.Input`
 :py:class:`beat.backend.python.inputs.InputGroup`
 """
-
-
-from beat.backend.python.inputs import InputList  # noqa
 from beat.backend.python.inputs import Input  # noqa
 from beat.backend.python.inputs import InputGroup  # noqa
+from beat.backend.python.inputs import InputList  # noqa

beat/core/library.py

@@ -44,14 +44,13 @@ Validation for libraries
 Forward imported from :py:mod:`beat.backend.python.library`:
 :py:class:`beat.backend.python.library.Storage`
 """
-
 import six
 
-from . import schema
-from . import prototypes
-
-from beat.backend.python.library import Storage
 from beat.backend.python.library import Library as BackendLibrary
+from beat.backend.python.library import Storage
+
+from . import prototypes
+from . import schema
 
 
 class Library(BackendLibrary):

beat/core/outputs.py

@@ -44,8 +44,6 @@ Forward imported from :py:mod:`beat.backend.python.outputs`:
 :py:class:`beat.backend.python.outputs.Output`
 :py:class:`beat.backend.python.outputs.OutputList`
 """
-
-
-from beat.backend.python.outputs import SynchronizationListener  # noqa
 from beat.backend.python.outputs import Output  # noqa
 from beat.backend.python.outputs import OutputList  # noqa
+from beat.backend.python.outputs import SynchronizationListener  # noqa

beat/core/plotter.py

@@ -41,18 +41,17 @@ plotter
 
 Validation for plotters
 """
-
 import os
-import six
 import sys
 
-from . import dataformat
+import six
+
 from . import algorithm
-from . import schema
+from . import dataformat
+from . import loader
 from . import prototypes
+from . import schema
 from . import utils
-from . import loader
-
 
 # ----------------------------------------------------------
 

beat/core/plotterparameter.py

@@ -41,13 +41,12 @@ plotterparameter
 
 Validation for plotterparameters
 """
-
 import simplejson as json
 
-from . import schema
+from . import plotter
 from . import prototypes
+from . import schema
 from . import utils
-from . import plotter
 
 
 class Storage(utils.Storage):

beat/core/protocoltemplate.py

@@ -44,16 +44,15 @@ Validation of protocoltemplate
 Forward importing from :py:mod:`beat.backend.python.protocoltemplate`:
 :py:class:`beat.backend.python.protocoltemplate.Storage`
 """
-
 import six
 
-from . import schema
-from . import prototypes
-
-from beat.backend.python.protocoltemplate import Storage
 from beat.backend.python.protocoltemplate import (
     ProtocolTemplate as BackendProtocolTemplate,
 )
+from beat.backend.python.protocoltemplate import Storage
+
+from . import prototypes
+from . import schema
 
 
 class ProtocolTemplate(BackendProtocolTemplate):

beat/core/prototypes/database.py

-import numpy
-
 from collections import namedtuple
 
+import numpy
+
 from beat.backend.python.database import View
 
 

beat/core/schema/__init__.py

@@ -34,14 +34,12 @@
 ###################################################################################
 
 """Schema validation for BEAT JSON I/O"""
-
-
 import os
-import pkg_resources
 
-import six
-import simplejson as json
 import jsonschema
+import pkg_resources
+import simplejson as json
+import six
 
 from beat.backend.python.utils import error_on_duplicate_key_hook
 
@@ -98,7 +96,7 @@ def load_schema(schema_name, version=1):
         try:
             schema = json.loads(data)
         except json.errors.JSONDecodeError:
-            print("Invalid json:\n {data}".format(data))
+            print("Invalid json:\n {}".format(data))
             raise
 
     basedir = os.path.realpath(os.path.dirname(fname))

beat/core/scripts/benchmark.py

+import time
+
+import numpy as np
+import pkg_resources
+import zmq
+
 from ..agent import MessageHandler
-from ..dataformat import DataFormat
 from ..data import DataSource
-from ..inputs import RemoteInput
+from ..dataformat