[backend] Pre-commit cleanup

beat/backend/python/algorithm.py

@@ -42,20 +42,19 @@ algorithm
 Validation for algorithms
 """
 
+import logging
 import os
 import sys
-import logging
 
-import six
 import numpy
 import simplejson as json
+import six
 
 from . import dataformat
 from . import library
 from . import loader
 from . import utils
 
-
 logger = logging.getLogger(__name__)
 
 

beat/backend/python/data.py

@@ -42,23 +42,24 @@ data
 Data I/O classes and functions
 """
 
+import abc
+import glob
+import logging
 import os
 import re
-import glob
-import simplejson as json
 import time
-import abc
-import zmq
-import logging
-import six
 
-from functools import reduce
 from collections import namedtuple
+from functools import reduce
+
+import simplejson as json
+import six
+import zmq
 
-from .hash import hashFileContents
-from .dataformat import DataFormat
 from .algorithm import Algorithm
+from .dataformat import DataFormat
 from .exceptions import RemoteException
+from .hash import hashFileContents
 
 logger = logging.getLogger(__name__)
 
@@ -107,8 +108,8 @@ def mixDataIndices(list_of_data_indices):
     for index in range(start, end + 1):
         done = False
 
-        for l in list_of_data_indices:
-            for indices in l:
+        for data_indices in list_of_data_indices:
+            for indices in data_indices:
                 if indices[1] == index:
                     result.append((current_start, index))
                     current_start = index + 1

beat/backend/python/data_loaders.py

@@ -46,7 +46,6 @@ import six
 
 from .data import mixDataIndices
 
-
 # ----------------------------------------------------------
 
 

beat/backend/python/database.py

@@ -42,23 +42,22 @@ database
 Validation of databases
 """
 
+import itertools
 import os
 import sys
 
-import six
-import simplejson as json
-import itertools
-import numpy as np
 from collections import namedtuple
 
+import numpy as np
+import simplejson as json
+import six
+
 from . import loader
 from . import utils
-
-from .protocoltemplate import ProtocolTemplate
 from .dataformat import DataFormat
-from .outputs import OutputList
 from .exceptions import OutputError
-
+from .outputs import OutputList
+from .protocoltemplate import ProtocolTemplate
 
 # ----------------------------------------------------------
 

beat/backend/python/dataformat.py

@@ -42,17 +42,16 @@ dataformat
 Validation and parsing for dataformats
 """
 
-import re
 import copy
+import re
 
-import six
 import numpy
 import simplejson as json
+import six
 
 from . import utils
 from .baseformat import baseformat
 
-
 # ----------------------------------------------------------
 
 

beat/backend/python/hash.py

@@ -42,13 +42,13 @@ hash
 Various functions for hashing platform contributions and others
 """
 
-import hashlib
-import simplejson
 import collections
 import copy
-import six
+import hashlib
 import os
 
+import simplejson
+import six
 
 # ----------------------------------------------------------
 
@@ -59,9 +59,9 @@ def _sha256(s):
     """
     if isinstance(s, six.string_types):
         try:
-            s = six.u(s).encode('utf-8')
+            s = six.u(s).encode("utf-8")
         except Exception:
-            s = s.encode('utf-8')
+            s = s.encode("utf-8")
     return hashlib.sha256(s).hexdigest()
 
 
@@ -71,14 +71,14 @@ def _sha256(s):
 def _stringify(dictionary):
     names = sorted(dictionary.keys())
 
-    converted_dictionary = '{'
+    converted_dictionary = "{"
     for name in names:
         converted_dictionary += '"%s":%s,' % (name, str(dictionary[name]))
 
     if len(converted_dictionary) > 1:
         converted_dictionary = converted_dictionary[:-1]
 
-    converted_dictionary += '}'
+    converted_dictionary += "}"
 
     return converted_dictionary
 
@@ -87,13 +87,13 @@ def _stringify(dictionary):
 
 
 def _compact(text):
-    return text.replace(' ', '').replace('\n', '')
+    return text.replace(" ", "").replace("\n", "")
 
 
 # ----------------------------------------------------------
 
 
-def toPath(hash, suffix='.data'):
+def toPath(hash, suffix=".data"):
     """ Returns the path on disk which corresponds to the hash given.
 
     Parameters:
@@ -159,7 +159,7 @@ def hashJSON(contents, description):
         contents = copy.deepcopy(contents)  # temporary copy
         del contents[description]
     contents = simplejson.dumps(contents, sort_keys=True)
-    return hashlib.sha256(contents.encode('utf-8')).hexdigest()
+    return hashlib.sha256(contents.encode("utf-8")).hexdigest()
 
 
 # ----------------------------------------------------------
@@ -175,11 +175,14 @@ def hashJSONFile(path, description):
     """
 
     try:
-        with open(path, 'rb') as f:
+        with open(path, "rb") as f:
             # preserve order
-            return hashJSON(simplejson.loads(f.read().decode('utf-8'),
-                            object_pairs_hook=collections.OrderedDict),
-                            description)
+            return hashJSON(
+                simplejson.loads(
+                    f.read().decode("utf-8"), object_pairs_hook=collections.OrderedDict
+                ),
+                description,
+            )
     except simplejson.JSONDecodeError:
         # falls back to normal file content hashing
         return hashFileContents(path)
@@ -195,9 +198,9 @@ def hashFileContents(path):
         str: hash
     """
 
-    with open(path, 'rb') as f:
+    with open(path, "rb") as f:
         sha256 = hashlib.sha256()
-        for chunk in iter(lambda: f.read(sha256.block_size * 1000), b''):
+        for chunk in iter(lambda: f.read(sha256.block_size * 1000), b""):
             sha256.update(chunk)
 
         return sha256.hexdigest()
@@ -218,9 +221,14 @@ def hashDataset(database_name, protocol_name, set_name):
         str: hash
     """
 
-    s = _compact("""{
+    s = (
+        _compact(
+            """{
         "database": "%s",
         "protocol": "%s",
         "set": "%s"
-}""") % (database_name, protocol_name, set_name)
+}"""
+        )
+        % (database_name, protocol_name, set_name)
+    )
     return hash(s)

beat/backend/python/helpers.py

@@ -42,24 +42,24 @@ helpers
 This module implements various helper methods and classes
 """
 
-import os
 import errno
 import logging
+import os
 
+from .algorithm import Algorithm
+from .data import CachedDataSink
 from .data import CachedDataSource
 from .data import RemoteDataSource
-from .data import CachedDataSink
 from .data import getAllFilenames
-from .data_loaders import DataLoaderList
 from .data_loaders import DataLoader
-from .inputs import InputList
+from .data_loaders import DataLoaderList
 from .inputs import Input
 from .inputs import InputGroup
-from .outputs import SynchronizationListener
-from .outputs import OutputList
+from .inputs import InputList
 from .outputs import Output
+from .outputs import OutputList
 from .outputs import RemotelySyncedOutput
-from .algorithm import Algorithm
+from .outputs import SynchronizationListener
 
 logger = logging.getLogger(__name__)
 

beat/backend/python/inputs.py

@@ -46,7 +46,6 @@ from functools import reduce
 
 import six
 
-
 # ----------------------------------------------------------
 
 

beat/backend/python/library.py

@@ -43,12 +43,12 @@ Validation for libraries
 """
 
 import os
+
 import simplejson as json
 
 from . import loader
 from . import utils
 
-
 # ----------------------------------------------------------
 
 

beat/backend/python/loader.py

@@ -44,10 +44,10 @@ executor. Safe in this context means that if the method raises an
 exception, it will catch it and return in a suitable form to the caller.
 """
 
-import sys
-import six
 import imp
+import sys
 
+import six
 
 # ----------------------------------------------------------
 
@@ -79,11 +79,11 @@ def load_module(name, path, uses):
 
     # loads used modules
     for k, v in uses.items():
-        retval.__dict__[k] = load_module(k, v['path'], v['uses'])
+        retval.__dict__[k] = load_module(k, v["path"], v["uses"])
 
     # execute the module code on the context of previously import modules
     with open(path, "rb") as f:
-      exec(compile(f.read(), path, 'exec'), retval.__dict__)
+        exec(compile(f.read(), path, "exec"), retval.__dict__)  # nosec
 
     return retval
 
@@ -118,7 +118,7 @@ def run(obj, method, exc=None, *args, **kwargs):
     """
 
     try:
-        if method == '__new__':
+        if method == "__new__":
             return obj(*args, **kwargs)
 
         return getattr(obj, method)(*args, **kwargs)

beat/backend/python/protocoltemplate.py

@@ -44,10 +44,8 @@ Validation of database protocol templates
 
 import simplejson as json
 
-from .dataformat import DataFormat
-
 from . import utils
-
+from .dataformat import DataFormat
 
 # ----------------------------------------------------------
 

beat/backend/python/stats.py

@@ -89,17 +89,14 @@ def io_statistics(configuration, input_list=None, output_list=None):
             write_time += duration
             blocks_written += output.nb_data_blocks_written
 
-            if 'result' in configuration:
-                hash = configuration['result']['path'].replace('/', '')
+            if "result" in configuration:
+                hash = configuration["result"]["path"].replace("/", "")
             else:
-                hash = configuration['outputs'][output.name]['path'].replace(
-                    '/', '')
+                hash = configuration["outputs"][output.name]["path"].replace("/", "")
 
-            files.append(dict(
-                hash=hash,
-                size=size,
-                blocks=output.nb_data_blocks_written,
-            ))
+            files.append(
+                dict(hash=hash, size=size, blocks=output.nb_data_blocks_written,)
+            )
 
     # Result
     return dict(
@@ -128,13 +125,13 @@ def update(statistics, additional_statistics):
     """
 
     for k in statistics.keys():
-        if k == 'files':
+        if k == "files":
             continue
 
         for k2 in statistics[k].keys():
             statistics[k][k2] += additional_statistics[k][k2]
 
-    if 'files' in statistics:
-        statistics['files'].extend(additional_statistics.get('files', []))
+    if "files" in statistics:
+        statistics["files"].extend(additional_statistics.get("files", []))
     else:
-        statistics['files'] = additional_statistics.get('files', [])
+        statistics["files"] = additional_statistics.get("files", [])

beat/backend/python/utils.py

@@ -42,16 +42,16 @@ utils
 This module implements helper classes and functions.
 """
 
+import collections
 import os
 import shutil
-import collections
+
 import numpy
 import simplejson
 import six
 
 from . import hash
 
-
 # ----------------------------------------------------------