Merge branch 'cleanup' into 'master'

Code cleanup

See merge request !62

.pre-commit-config.yaml

@@ -6,12 +6,14 @@ repos:
     hooks:
     - id: black
       language_version: python3.6
+      exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
+        exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
       - id: check-added-large-files
       - id: check-docstring-first
       - id: flake8
@@ -21,7 +23,7 @@ repos:
     rev: 'master' # Update me!
     hooks:
       - id: bandit
-        exclude: beat/editor/test
+        exclude: beat/editor/test|beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: local
     hooks:
       - id: sphinx-build

beat/core/algorithm.py

@@ -66,7 +66,9 @@ from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
 
 def load_algorithm_prototype(prefix):
 
-    prototype_data = pkg_resources.resource_string(__name__, "prototypes/algorithm.json")
+    prototype_data = pkg_resources.resource_string(
+        __name__, "prototypes/algorithm.json"
+    )
     if sys.version_info < (3, 6):
         prototype_data = prototype_data.decode("utf-8")
 

beat/core/baseformat.py

@@ -42,4 +42,4 @@ baseformat
 Froward imports from :py:mod:`beat.backend.python.baseformat`
 """
 
-from beat.backend.python.baseformat import *
+from beat.backend.python.baseformat import *  # noqa

beat/core/data.py

@@ -54,15 +54,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
 :py:func:`beat.backend.python.data.foundSplitRanges`
 """
 
-from beat.backend.python.data import mixDataIndices
-from beat.backend.python.data import getAllFilenames
-from beat.backend.python.data import DataSource
-from beat.backend.python.data import CachedDataSource
-from beat.backend.python.data import DatabaseOutputDataSource
-from beat.backend.python.data import RemoteDataSource
-from beat.backend.python.data import DataSink
-from beat.backend.python.data import CachedDataSink
-from beat.backend.python.data import StdoutDataSink
-from beat.backend.python.data import load_data_index
-from beat.backend.python.data import load_data_index_db
-from beat.backend.python.data import foundSplitRanges
+from beat.backend.python.data import mixDataIndices  # noqa
+from beat.backend.python.data import getAllFilenames  # noqa
+from beat.backend.python.data import DataSource  # noqa
+from beat.backend.python.data import CachedDataSource  # noqa
+from beat.backend.python.data import DatabaseOutputDataSource  # noqa
+from beat.backend.python.data import RemoteDataSource  # noqa
+from beat.backend.python.data import DataSink  # noqa
+from beat.backend.python.data import CachedDataSink  # noqa
+from beat.backend.python.data import StdoutDataSink  # noqa
+from beat.backend.python.data import load_data_index  # noqa
+from beat.backend.python.data import load_data_index_db  # noqa
+from beat.backend.python.data import foundSplitRanges  # noqa

beat/core/data_loaders.py

@@ -46,6 +46,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
 """
 
 
-from beat.backend.python.data_loaders import DataLoaderList
-from beat.backend.python.data_loaders import DataLoader
-from beat.backend.python.data_loaders import DataView
+from beat.backend.python.data_loaders import DataLoaderList  # noqa
+from beat.backend.python.data_loaders import DataLoader  # noqa
+from beat.backend.python.data_loaders import DataView  # noqa

beat/core/dataformat.py

@@ -45,18 +45,14 @@ Forward importing from :py:mod:`beat.backend.python.dataformat`:
 :py:class:`beat.backend.python.dataformat.Storage`
 """
 
-import os
 import copy
-
 import six
-import simplejson
 
 from . import schema
 from . import prototypes
 from . import utils
-from .baseformat import baseformat
 
-from beat.backend.python.dataformat import Storage
+from beat.backend.python.dataformat import Storage  # noqa
 from beat.backend.python.dataformat import DataFormat as BackendDataFormat
 
 
@@ -132,7 +128,7 @@ class DataFormat(BackendDataFormat):
         if data is None:  # loads prototype and validates it
 
             self.data, self.errors = prototypes.load("dataformat")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
 
         else:
 

beat/core/dock.py

@@ -49,7 +49,7 @@ import socket
 import tempfile
 import time
 import docker
-import subprocess as sp
+import subprocess as sp  # nosec
 import logging
 
 from beat.core import stats
@@ -140,14 +140,14 @@ class Host(object):
                     self.processing_environments.keys(),
                 )
             )[0]
-        except:
+        except IndexError:
             try:
                 return list(
                     filter(
                         lambda x: x.startswith(name + " ("), self.db_environments.keys()
                     )
                 )[0]
-            except:
+            except IndexError:
                 return None
 
     @property
@@ -245,10 +245,10 @@ class Host(object):
                                 replacement = True
                             else:
                                 keep = True
-                        except:
+                        except Exception:
                             replacement = True
 
-                    except:
+                    except Exception:
                         keep = True
 
             elif new_version is not None:
@@ -493,7 +493,7 @@ class Host(object):
 
         try:
             return simplejson.loads(stdout)[0]["State"]["Status"]
-        except:
+        except Exception:
             return None
 
     def logs(self, container):
@@ -569,7 +569,7 @@ class Host(object):
             status = self.wait(container)
             output = self.logs(container)
 
-        except Exception as e:
+        except Exception:
             return 1, None
 
         finally:

beat/core/environments.py

@@ -79,7 +79,7 @@ def enumerate_packages(host, environment_name):
     package_lines = output.split("\n")
 
     for package_line in package_lines:
-        information = re.split("\s+", package_line)
+        information = re.split(r"\s+", package_line)
         if len(information) == 4:
             packages.append({"name": information[0], "version": information[1]})
 

beat/core/execution/__init__.py

@@ -33,7 +33,8 @@
 #                                                                                 #
 ###################################################################################
 
+# flake8: noqa
 
 from .docker import DockerExecutor
 from .local import LocalExecutor
-from .subprocess import SubprocessExecutor
+from .subprocess import SubprocessExecutor  # nosec

beat/core/execution/base.py

@@ -44,14 +44,9 @@ Execution utilities
 
 
 import os
-import sys
 import glob
 import collections
-
 import logging
-
-logger = logging.getLogger(__name__)
-
 import simplejson
 
 from .. import schema
@@ -62,6 +57,9 @@ from .. import stats
 from beat.backend.python.helpers import convert_experiment_configuration_to_container
 
 
+logger = logging.getLogger(__name__)
+
+
 class BaseExecutor(object):
     """Executors runs the code given an execution block information
 

beat/core/execution/docker.py

@@ -47,8 +47,6 @@ import shutil
 import logging
 import requests
 import simplejson
-import zmq
-import docker
 
 from beat.backend.python.execution import MessageHandler
 
@@ -203,7 +201,7 @@ class DockerExecutor(RemoteExecutor):
         # Determine the docker image to use for the databases
         try:
             databases_environment = self.host.db2docker(database_paths.keys())
-        except:
+        except Exception:
             raise RuntimeError(
                 "No environment found for the databases `%s' "
                 "- available environments are %s"

beat/core/execution/local.py

@@ -49,24 +49,17 @@ import tempfile
 import shutil
 import zmq
 import time
-
 import logging
 
-logger = logging.getLogger(__name__)
-
-import simplejson
-
-from .base import BaseExecutor
-
-from beat.backend.python.helpers import create_inputs_from_configuration
-from beat.backend.python.helpers import create_outputs_from_configuration
-from beat.backend.python.helpers import AccessMode
-
 from beat.backend.python.execution import AlgorithmExecutor
 from beat.backend.python.execution import MessageHandler
 from beat.backend.python.execution import LoopExecutor
 from beat.backend.python.execution import LoopMessageHandler
 
+from .base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
 
 class LocalExecutor(BaseExecutor):
     """LocalExecutor runs the code given an execution block information

beat/core/execution/remote.py

@@ -42,17 +42,16 @@ remote
 Execution utilities
 """
 
-import zmq
-
 import logging
 
-logger = logging.getLogger(__name__)
 
 from .base import BaseExecutor
 
 from beat.backend.python.helpers import create_inputs_from_configuration
 from beat.backend.python.helpers import create_outputs_from_configuration
-from beat.backend.python.helpers import AccessMode
+
+
+logger = logging.getLogger(__name__)
 
 
 class RemoteExecutor(BaseExecutor):

beat/core/execution/subprocess.py

@@ -41,16 +41,12 @@ subprocess
 Execution utilities
 """
 
-from __future__ import absolute_import
-
 import os
 import shutil
 import logging
 import sys
-import subprocess as sp
+import subprocess as sp  # nosec
 import tempfile
-import requests
-import zmq
 
 from beat.backend.python.execution import MessageHandler
 
@@ -415,7 +411,7 @@ class SubprocessExecutor(RemoteExecutor):
             algorithm_process.kill()
             status = algorithm_process.wait()
 
-        except:
+        except Exception:
             algorithm_process.kill()
             status = algorithm_process.wait()
 

beat/core/experiment.py

@@ -231,9 +231,9 @@ class Experiment(object):
         if data is None:  # loads prototype and validates it
 
             experiment_data, self.errors = prototypes.load("experiment")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
             toolchain_data, self.errors = prototypes.load("toolchain")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
 
         elif isinstance(data, (tuple, list)):  # the user has passed a tuple
 
@@ -477,7 +477,7 @@ class Experiment(object):
                 self.errors.append(
                     "/loop/%s/nb_slots: you have set the number "
                     "of slots for algorithm `%s' to %d, but it is not "
-                    "splittable" % (analyzername, thisalgo.name, loop["nb_slots"])
+                    "splittable" % (algoname, thisalgo.name, loop["nb_slots"])
                 )
 
             # check parameter consistence
@@ -740,7 +740,7 @@ class Experiment(object):
                 from_dtype = self.algorithms[block["algorithm"]].output_map[algout]
                 from_name = "block"
             else:
-                self.errors.append("Unknown endpoint %s" % to_endpt[0])
+                self.errors.append("Unknown endpoint %s" % from_endpt[0])
                 continue
 
             to_endpt = connection["to"].split(".", 1)
@@ -869,7 +869,7 @@ class Experiment(object):
             # for the grouping properties for the inputs
 
             # create channel groups
-            chain_in = collections.Counter(input_connections)
+            chain_groups = collections.Counter(input_connections)
 
             # now check the algorithm for conformance
             algo_groups = self.algorithms[self.analyzers[name]["algorithm"]].groups
@@ -926,7 +926,7 @@ class Experiment(object):
         )
 
         # makes sure we don't have multiple incomming connections
-        assert len(_connections) == len(connections), (
+        assert len(_connections) == len(connections), (  # nosec
             "detected multiple input "
             "connections for block `%s' on experiment `%s'" % (name, self.label)
         )
@@ -975,7 +975,6 @@ class Experiment(object):
                 # then go one by one generating the input **and** output hashes
                 # until all is done.
 
-                block_config = self.blocks[block]
                 retval[algo_endpt] = {
                     "from": "%s.%s" % (block, output),
                     "channel": channel,
@@ -1181,8 +1180,6 @@ class Experiment(object):
     def dot_diagram(self):
         """Returns a dot diagram representation of the experiment"""
 
-        from .drawing import create_port_table
-
         title = "Experiment: %s" % self.label
 
         def __label_callback(type, name):

beat/core/hash.py

@@ -48,8 +48,8 @@ import collections
 
 import simplejson
 
-from beat.backend.python.hash import *
-from beat.backend.python.hash import _sha256
+from beat.backend.python.hash import *  # noqa
+from beat.backend.python.hash import _sha256  # noqa
 from beat.backend.python.hash import _stringify
 from beat.backend.python.hash import _compact
 
@@ -164,7 +164,7 @@ def hashJSONStr(contents, description):
     """
 
     try:
-        return hashJSON(
+        return hashJSON(  # noqa
             simplejson.loads(contents, object_pairs_hook=collections.OrderedDict),
             description,
         )  # preserve order

beat/core/inputs.py

@@ -47,6 +47,6 @@ Forward imported from :py:mod:`beat.backend.python.inputs`:
 """
 
 
-from beat.backend.python.inputs import InputList
-from beat.backend.python.inputs import Input
-from beat.backend.python.inputs import InputGroup
+from beat.backend.python.inputs import InputList  # noqa
+from beat.backend.python.inputs import Input  # noqa
+from beat.backend.python.inputs import InputGroup  # noqa

beat/core/library.py

@@ -147,7 +147,7 @@ class Library(BackendLibrary):
         # At this point, `data' can be a dictionary or ``None``
         if data is None:  # loads the default declaration for an library
             self.data, self.errors = prototypes.load("library")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
         else:  # just assign it
             # this runs basic validation, including JSON loading if required
             self.data, self.errors = schema.validate("library", data)

beat/core/loader.py

@@ -43,4 +43,4 @@ Forward imports from :py:mod:`beat.backend.python.loader`
 """
 
 
-from beat.backend.python.loader import *
+from beat.backend.python.loader import *  # noqa

beat/core/outputs.py

@@ -46,6 +46,6 @@ Forward imported from :py:mod:`beat.backend.python.outputs`:
 """
 
 
-from beat.backend.python.outputs import SynchronizationListener
-from beat.backend.python.outputs import Output
-from beat.backend.python.outputs import OutputList
+from beat.backend.python.outputs import SynchronizationListener  # noqa
+from beat.backend.python.outputs import Output  # noqa
+from beat.backend.python.outputs import OutputList  # noqa

beat/core/plotter.py

@@ -44,6 +44,7 @@ Validation for plotters
 
 import os
 import six
+import sys
 
 from . import dataformat
 from . import algorithm
@@ -227,7 +228,7 @@ class Plotter(object):
         # At this point, `data' can be a dictionary or ``None``
         if data is None:  # loads the default declaration for an algorithm
             self.data, self.errors = prototypes.load("plotter")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
         else:  # just assign it
             # this runs basic validation, including JSON loading if required
             self.data, self.errors = schema.validate("plotter", data)
@@ -400,7 +401,7 @@ class Plotter(object):
                     self.uses_dict(),
                 ),
             )
-        except Exception as e:
+        except Exception:
             if exc is not None:
                 type, value, traceback = sys.exc_info()
                 six.reraise(exc, exc(value), traceback)

beat/core/prototypes/__init__.py

@@ -37,8 +37,6 @@
 
 import pkg_resources
 
-import simplejson
-
 from .. import schema
 
 

beat/core/prototypes/algorithm.py

 class Algorithm:
-
     def process(self, inputs, data_loaders, outputs):
 
         # TODO: Implement this algorithm

beat/core/prototypes/plotter.py

 class Plotter:
-
     def process(self, input):
 
         # TODO: Implement this plotter
 
-        #return image as a string
+        # return image as a string
         pass

beat/core/prototypes/view.py

 class View:
+    def setup(
+        self,
+        root_folder,
+        outputs,
+        parameters,
+        force_start_index=None,
+        force_end_index=None,
+    ):
+        """Initializes the database"""
 
-  def setup(self, root_folder, outputs, parameters, force_start_index=None, force_end_index=None):
-    '''Initializes the database'''
+        return True
 
-    return True
+    def done(self):
+        """Should return ``True``, when data is finished"""
 
+        return True
 
-  def done(self):
-    '''Should return ``True``, when data is finished'''
+    def next(self):
+        """Loads the next data block on ``outputs``"""
 
-    return True
-
-
-  def next(self):
-    '''Loads the next data block on ``outputs``'''
-
-    return True
+        return True

beat/core/schema/algorithm/2.json

@@ -76,4 +76,4 @@
   },
 
   "required": ["type", "api_version", "schema_version"]
-}
\ No newline at end of file
+}

beat/core/schema/algorithm/3.json

@@ -160,4 +160,4 @@
   },
 
   "required": ["type", "api_version", "schema_version"]
-}
\ No newline at end of file
+}

beat/core/schema/toolchain/common.json

@@ -220,4 +220,4 @@
       }
     }
   }
-}
\ No newline at end of file
+}

beat/core/scripts/benchmark.py

@@ -2,7 +2,6 @@ from ..agent import MessageHandler
 from ..dataformat import DataFormat
 from ..data import DataSource
 from ..inputs import RemoteInput
-from ..inputs import RemoteException
 from ..inputs import Input
 from ..inputs import InputGroup
 from ..inputs import InputList
@@ -12,18 +11,16 @@ import time
 import zmq
 
 
-prefix = pkg_resources.resource_filename(__name__, '../test/prefix')
+prefix = pkg_resources.resource_filename(__name__, "../test/prefix")
 print(prefix)
 
 
-
 class CustomDataSource(DataSource):
-
     def __init__(self, nb_data_units, dataformat):
         self.nb_data_units = nb_data_units
         self.current = 0
         self.dataformat = dataformat
-        self.file = open('benchmark.data', 'rb')
+        self.file = open("benchmark.data", "rb")
         self.unpack = True
 
     def next(self):
@@ -50,20 +47,20 @@ class CustomDataSource(DataSource):
 
 
 def main():
-    dataformat = DataFormat(prefix, 'user/empty_1d_array_of_integers/1')
+    dataformat = DataFormat(prefix, "user/empty_1d_array_of_integers/1")
 
-    data = dataformat.type(value=np.random.randint(100000, size=(1000000,), dtype=np.int32))
+    data = dataformat.type(
+        value=np.random.randint(100000, size=(1000000,), dtype=np.int32)
+    )
 
-    with open('benchmark.data', 'wb') as f:
+    with open("benchmark.data", "wb") as f:
         f.write(data.pack())
 
-
-    #-----------------------
-
+    # -----------------------
 
     data_source = CustomDataSource(10000, dataformat)
 
-    print('Nb Data units:       %d' % data_source.nb_data_units)
+    print("Nb Data units:       %d" % data_source.nb_data_units)
 
     t1 = time.time()
 
@@ -72,12 +69,14 @@ def main():
 
     t2 = time.time()
 
-    print('Datasource (unpack): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
+    print(
+        "Datasource (unpack): %.3fs (%.3fms/unit)"
+        % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
+    )
 
     data_source.reset()
 
-
-    #-----------------------
+    # -----------------------
 
     data_source.unpack = False
 
@@ -88,16 +87,16 @@ def main():
 
     t2 = time.time()
 
-    print('Datasource (packed): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
+    print(
+        "Datasource (packed): %.3fs (%.3fms/unit)"
+        % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
+    )
 
     data_source.reset()
 
+    # -----------------------
 
-    #-----------------------
-
-
-    input = Input('a', 'user/empty_1d_array_of_integers/1', data_source)
-
+    input = Input("a", "user/empty_1d_array_of_integers/1", data_source)
 
     t1 = time.time()
 
@@ -106,13 +105,14 @@ def main():
 
     t2 = time.time()
 
-    print('Input (unpack):      %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
+    print(
+        "Input (unpack):      %.3fs (%.3fms/unit)"
+        % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
+    )
 
     data_source.reset()
 
-
-    #-----------------------
-
+    # -----------------------
 
     data_source.unpack = False
 
@@ -123,37 +123,36 @@ def main():
 
     t2 = time.time()
 
-    print('Input (packed):      %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
+    print(
+        "Input (packed):      %.3fs (%.3fms/unit)"
+        % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
+    )
 
     data_source.reset()
 
+    # -----------------------
 
-    #-----------------------
-
-
-    group = InputGroup('channel')
+    group = InputGroup("channel")
     group.add(input)
 
     input_list = InputList()
     input_list.add(group)
 
-
     server_context = zmq.Context()
     server_socket = server_context.socket(zmq.PAIR)
-    address = 'tcp://127.0.0.1'
+    address = "tcp://127.0.0.1"
     port = server_socket.bind_to_random_port(address)
-    address += ':%d' % port
+    address += ":%d" % port
 
     message_handler = MessageHandler(input_list, server_context, server_socket)
 
-
     client_context = zmq.Context()
     client_socket = client_context.socket(zmq.PAIR)
     client_socket.connect(address)