Merge branch 'cleanup' into 'master'

Code cleanup

See merge request !62

.pre-commit-config.yaml

@@ -6,12 +6,14 @@ repos:
     hooks:
     - id: black
       language_version: python3.6
+      exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
+        exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
       - id: check-added-large-files
       - id: check-docstring-first
       - id: flake8
@@ -21,7 +23,7 @@ repos:
     rev: 'master' # Update me!
     hooks:
       - id: bandit
-        exclude: beat/editor/test
+        exclude: beat/editor/test|beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
   - repo: local
     hooks:
       - id: sphinx-build

beat/core/algorithm.py

@@ -66,7 +66,9 @@ from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
 
 def load_algorithm_prototype(prefix):
 
-    prototype_data = pkg_resources.resource_string(__name__, "prototypes/algorithm.json")
+    prototype_data = pkg_resources.resource_string(
+        __name__, "prototypes/algorithm.json"
+    )
     if sys.version_info < (3, 6):
         prototype_data = prototype_data.decode("utf-8")
 

beat/core/baseformat.py

@@ -42,4 +42,4 @@ baseformat
 Froward imports from :py:mod:`beat.backend.python.baseformat`
 """
 
-from beat.backend.python.baseformat import *
+from beat.backend.python.baseformat import *  # noqa

beat/core/data.py

@@ -54,15 +54,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
 :py:func:`beat.backend.python.data.foundSplitRanges`
 """
 
-from beat.backend.python.data import mixDataIndices
-from beat.backend.python.data import getAllFilenames
-from beat.backend.python.data import DataSource
-from beat.backend.python.data import CachedDataSource
-from beat.backend.python.data import DatabaseOutputDataSource
-from beat.backend.python.data import RemoteDataSource
-from beat.backend.python.data import DataSink
-from beat.backend.python.data import CachedDataSink
-from beat.backend.python.data import StdoutDataSink
-from beat.backend.python.data import load_data_index
-from beat.backend.python.data import load_data_index_db
-from beat.backend.python.data import foundSplitRanges
+from beat.backend.python.data import mixDataIndices  # noqa
+from beat.backend.python.data import getAllFilenames  # noqa
+from beat.backend.python.data import DataSource  # noqa
+from beat.backend.python.data import CachedDataSource  # noqa
+from beat.backend.python.data import DatabaseOutputDataSource  # noqa
+from beat.backend.python.data import RemoteDataSource  # noqa
+from beat.backend.python.data import DataSink  # noqa
+from beat.backend.python.data import CachedDataSink  # noqa
+from beat.backend.python.data import StdoutDataSink  # noqa
+from beat.backend.python.data import load_data_index  # noqa
+from beat.backend.python.data import load_data_index_db  # noqa
+from beat.backend.python.data import foundSplitRanges  # noqa

beat/core/data_loaders.py

@@ -46,6 +46,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
 """
 
 
-from beat.backend.python.data_loaders import DataLoaderList
-from beat.backend.python.data_loaders import DataLoader
-from beat.backend.python.data_loaders import DataView
+from beat.backend.python.data_loaders import DataLoaderList  # noqa
+from beat.backend.python.data_loaders import DataLoader  # noqa
+from beat.backend.python.data_loaders import DataView  # noqa

beat/core/dataformat.py

@@ -45,18 +45,14 @@ Forward importing from :py:mod:`beat.backend.python.dataformat`:
 :py:class:`beat.backend.python.dataformat.Storage`
 """
 
-import os
 import copy
-
 import six
-import simplejson
 
 from . import schema
 from . import prototypes
 from . import utils
-from .baseformat import baseformat
 
-from beat.backend.python.dataformat import Storage
+from beat.backend.python.dataformat import Storage  # noqa
 from beat.backend.python.dataformat import DataFormat as BackendDataFormat
 
 
@@ -132,7 +128,7 @@ class DataFormat(BackendDataFormat):
         if data is None:  # loads prototype and validates it
 
             self.data, self.errors = prototypes.load("dataformat")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
 
         else:
 

beat/core/dock.py

@@ -49,7 +49,7 @@ import socket
 import tempfile
 import time
 import docker
-import subprocess as sp
+import subprocess as sp  # nosec
 import logging
 
 from beat.core import stats
@@ -140,14 +140,14 @@ class Host(object):
                     self.processing_environments.keys(),
                 )
             )[0]
-        except:
+        except IndexError:
             try:
                 return list(
                     filter(
                         lambda x: x.startswith(name + " ("), self.db_environments.keys()
                     )
                 )[0]
-            except:
+            except IndexError:
                 return None
 
     @property
@@ -245,10 +245,10 @@ class Host(object):
                                 replacement = True
                             else:
                                 keep = True
-                        except:
+                        except Exception:
                             replacement = True
 
-                    except:
+                    except Exception:
                         keep = True
 
             elif new_version is not None:
@@ -493,7 +493,7 @@ class Host(object):
 
         try:
             return simplejson.loads(stdout)[0]["State"]["Status"]
-        except:
+        except Exception:
             return None
 
     def logs(self, container):
@@ -569,7 +569,7 @@ class Host(object):
             status = self.wait(container)
             output = self.logs(container)
 
-        except Exception as e:
+        except Exception:
             return 1, None
 
         finally:

beat/core/environments.py

@@ -79,7 +79,7 @@ def enumerate_packages(host, environment_name):
     package_lines = output.split("\n")
 
     for package_line in package_lines:
-        information = re.split("\s+", package_line)
+        information = re.split(r"\s+", package_line)
         if len(information) == 4:
             packages.append({"name": information[0], "version": information[1]})
 

beat/core/execution/__init__.py

@@ -33,7 +33,8 @@
 #                                                                                 #
 ###################################################################################
 
+# flake8: noqa
 
 from .docker import DockerExecutor
 from .local import LocalExecutor
-from .subprocess import SubprocessExecutor
+from .subprocess import SubprocessExecutor  # nosec

beat/core/execution/base.py

@@ -44,14 +44,9 @@ Execution utilities
 
 
 import os
-import sys
 import glob
 import collections
-
 import logging
-
-logger = logging.getLogger(__name__)
-
 import simplejson
 
 from .. import schema
@@ -62,6 +57,9 @@ from .. import stats
 from beat.backend.python.helpers import convert_experiment_configuration_to_container
 
 
+logger = logging.getLogger(__name__)
+
+
 class BaseExecutor(object):
     """Executors runs the code given an execution block information
 

beat/core/execution/docker.py

@@ -47,8 +47,6 @@ import shutil
 import logging
 import requests
 import simplejson
-import zmq
-import docker
 
 from beat.backend.python.execution import MessageHandler
 
@@ -203,7 +201,7 @@ class DockerExecutor(RemoteExecutor):
         # Determine the docker image to use for the databases
         try:
             databases_environment = self.host.db2docker(database_paths.keys())
-        except:
+        except Exception:
             raise RuntimeError(
                 "No environment found for the databases `%s' "
                 "- available environments are %s"

beat/core/execution/local.py

@@ -49,24 +49,17 @@ import tempfile
 import shutil
 import zmq
 import time
-
 import logging
 
-logger = logging.getLogger(__name__)
-
-import simplejson
-
-from .base import BaseExecutor
-
-from beat.backend.python.helpers import create_inputs_from_configuration
-from beat.backend.python.helpers import create_outputs_from_configuration
-from beat.backend.python.helpers import AccessMode
-
 from beat.backend.python.execution import AlgorithmExecutor
 from beat.backend.python.execution import MessageHandler
 from beat.backend.python.execution import LoopExecutor
 from beat.backend.python.execution import LoopMessageHandler
 
+from .base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
 
 class LocalExecutor(BaseExecutor):
     """LocalExecutor runs the code given an execution block information

beat/core/execution/remote.py

@@ -42,17 +42,16 @@ remote
 Execution utilities
 """
 
-import zmq
-
 import logging
 
-logger = logging.getLogger(__name__)
 
 from .base import BaseExecutor
 
 from beat.backend.python.helpers import create_inputs_from_configuration
 from beat.backend.python.helpers import create_outputs_from_configuration
-from beat.backend.python.helpers import AccessMode
+
+
+logger = logging.getLogger(__name__)
 
 
 class RemoteExecutor(BaseExecutor):

beat/core/execution/subprocess.py

@@ -41,16 +41,12 @@ subprocess
 Execution utilities
 """
 
-from __future__ import absolute_import
-
 import os
 import shutil
 import logging
 import sys
-import subprocess as sp
+import subprocess as sp  # nosec
 import tempfile
-import requests
-import zmq
 
 from beat.backend.python.execution import MessageHandler
 
@@ -415,7 +411,7 @@ class SubprocessExecutor(RemoteExecutor):
             algorithm_process.kill()
             status = algorithm_process.wait()
 
-        except:
+        except Exception:
             algorithm_process.kill()
             status = algorithm_process.wait()
 

beat/core/experiment.py

@@ -231,9 +231,9 @@ class Experiment(object):
         if data is None:  # loads prototype and validates it
 
             experiment_data, self.errors = prototypes.load("experiment")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
             toolchain_data, self.errors = prototypes.load("toolchain")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
 
         elif isinstance(data, (tuple, list)):  # the user has passed a tuple
 
@@ -477,7 +477,7 @@ class Experiment(object):
                 self.errors.append(
                     "/loop/%s/nb_slots: you have set the number "
                     "of slots for algorithm `%s' to %d, but it is not "
-                    "splittable" % (analyzername, thisalgo.name, loop["nb_slots"])
+                    "splittable" % (algoname, thisalgo.name, loop["nb_slots"])
                 )
 
             # check parameter consistence
@@ -740,7 +740,7 @@ class Experiment(object):
                 from_dtype = self.algorithms[block["algorithm"]].output_map[algout]
                 from_name = "block"
             else:
-                self.errors.append("Unknown endpoint %s" % to_endpt[0])
+                self.errors.append("Unknown endpoint %s" % from_endpt[0])
                 continue
 
             to_endpt = connection["to"].split(".", 1)
@@ -869,7 +869,7 @@ class Experiment(object):
             # for the grouping properties for the inputs
 
             # create channel groups
-            chain_in = collections.Counter(input_connections)
+            chain_groups = collections.Counter(input_connections)
 
             # now check the algorithm for conformance
             algo_groups = self.algorithms[self.analyzers[name]["algorithm"]].groups
@@ -926,7 +926,7 @@ class Experiment(object):
         )
 
         # makes sure we don't have multiple incomming connections
-        assert len(_connections) == len(connections), (
+        assert len(_connections) == len(connections), (  # nosec
             "detected multiple input "
             "connections for block `%s' on experiment `%s'" % (name, self.label)
         )
@@ -975,7 +975,6 @@ class Experiment(object):
                 # then go one by one generating the input **and** output hashes
                 # until all is done.
 
-                block_config = self.blocks[block]
                 retval[algo_endpt] = {
                     "from": "%s.%s" % (block, output),
                     "channel": channel,
@@ -1181,8 +1180,6 @@ class Experiment(object):
     def dot_diagram(self):
         """Returns a dot diagram representation of the experiment"""
 
-        from .drawing import create_port_table
-
         title = "Experiment: %s" % self.label
 
         def __label_callback(type, name):

beat/core/hash.py

@@ -48,8 +48,8 @@ import collections
 
 import simplejson
 
-from beat.backend.python.hash import *
-from beat.backend.python.hash import _sha256
+from beat.backend.python.hash import *  # noqa
+from beat.backend.python.hash import _sha256  # noqa
 from beat.backend.python.hash import _stringify
 from beat.backend.python.hash import _compact
 
@@ -164,7 +164,7 @@ def hashJSONStr(contents, description):
     """
 
     try:
-        return hashJSON(
+        return hashJSON(  # noqa
             simplejson.loads(contents, object_pairs_hook=collections.OrderedDict),
             description,
         )  # preserve order

beat/core/inputs.py

@@ -47,6 +47,6 @@ Forward imported from :py:mod:`beat.backend.python.inputs`:
 """
 
 
-from beat.backend.python.inputs import InputList
-from beat.backend.python.inputs import Input
-from beat.backend.python.inputs import InputGroup
+from beat.backend.python.inputs import InputList  # noqa
+from beat.backend.python.inputs import Input  # noqa
+from beat.backend.python.inputs import InputGroup  # noqa

beat/core/library.py

@@ -147,7 +147,7 @@ class Library(BackendLibrary):
         # At this point, `data' can be a dictionary or ``None``
         if data is None:  # loads the default declaration for an library
             self.data, self.errors = prototypes.load("library")
-            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)
+            assert not self.errors, "\n  * %s" % "\n  *".join(self.errors)  # nosec
         else:  # just assign it
             # this runs basic validation, including JSON loading if required
             self.data, self.errors = schema.validate("library", data)

beat/core/loader.py

@@ -43,4 +43,4 @@ Forward imports from :py:mod:`beat.backend.python.loader`
 """
 
 
-from beat.backend.python.loader import *
+from beat.backend.python.loader import *  # noqa

beat/core/outputs.py

@@ -46,6 +46,6 @@ Forward imported from :py:mod:`beat.backend.python.outputs`:
 """
 
 
-from beat.backend.python.outputs import SynchronizationListener
-from beat.backend.python.outputs import Output
-from beat.backend.python.outputs import OutputList
+from beat.backend.python.outputs import SynchronizationListener  # noqa
+from beat.backend.python.outputs import Output  # noqa
+from beat.backend.python.outputs import OutputList  # noqa