Commit b8fd9960 authored by Samuel GAIST's avatar Samuel GAIST

[backend] Pre-commit cleanup

parent d1be9948
Pipeline #40902 passed with stage
in 5 minutes and 2 seconds
...@@ -42,20 +42,19 @@ algorithm ...@@ -42,20 +42,19 @@ algorithm
Validation for algorithms Validation for algorithms
""" """
import logging
import os import os
import sys import sys
import logging
import six
import numpy import numpy
import simplejson as json import simplejson as json
import six
from . import dataformat from . import dataformat
from . import library from . import library
from . import loader from . import loader
from . import utils from . import utils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
This diff is collapsed.
...@@ -42,23 +42,24 @@ data ...@@ -42,23 +42,24 @@ data
Data I/O classes and functions Data I/O classes and functions
""" """
import abc
import glob
import logging
import os import os
import re import re
import glob
import simplejson as json
import time import time
import abc
import zmq
import logging
import six
from functools import reduce
from collections import namedtuple from collections import namedtuple
from functools import reduce
import simplejson as json
import six
import zmq
from .hash import hashFileContents
from .dataformat import DataFormat
from .algorithm import Algorithm from .algorithm import Algorithm
from .dataformat import DataFormat
from .exceptions import RemoteException from .exceptions import RemoteException
from .hash import hashFileContents
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -107,8 +108,8 @@ def mixDataIndices(list_of_data_indices): ...@@ -107,8 +108,8 @@ def mixDataIndices(list_of_data_indices):
for index in range(start, end + 1): for index in range(start, end + 1):
done = False done = False
for l in list_of_data_indices: for data_indices in list_of_data_indices:
for indices in l: for indices in data_indices:
if indices[1] == index: if indices[1] == index:
result.append((current_start, index)) result.append((current_start, index))
current_start = index + 1 current_start = index + 1
......
...@@ -46,7 +46,6 @@ import six ...@@ -46,7 +46,6 @@ import six
from .data import mixDataIndices from .data import mixDataIndices
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -42,23 +42,22 @@ database ...@@ -42,23 +42,22 @@ database
Validation of databases Validation of databases
""" """
import itertools
import os import os
import sys import sys
import six
import simplejson as json
import itertools
import numpy as np
from collections import namedtuple from collections import namedtuple
import numpy as np
import simplejson as json
import six
from . import loader from . import loader
from . import utils from . import utils
from .protocoltemplate import ProtocolTemplate
from .dataformat import DataFormat from .dataformat import DataFormat
from .outputs import OutputList
from .exceptions import OutputError from .exceptions import OutputError
from .outputs import OutputList
from .protocoltemplate import ProtocolTemplate
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -42,17 +42,16 @@ dataformat ...@@ -42,17 +42,16 @@ dataformat
Validation and parsing for dataformats Validation and parsing for dataformats
""" """
import re
import copy import copy
import re
import six
import numpy import numpy
import simplejson as json import simplejson as json
import six
from . import utils from . import utils
from .baseformat import baseformat from .baseformat import baseformat
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -42,13 +42,13 @@ hash ...@@ -42,13 +42,13 @@ hash
Various functions for hashing platform contributions and others Various functions for hashing platform contributions and others
""" """
import hashlib
import simplejson
import collections import collections
import copy import copy
import six import hashlib
import os import os
import simplejson
import six
# ---------------------------------------------------------- # ----------------------------------------------------------
...@@ -59,9 +59,9 @@ def _sha256(s): ...@@ -59,9 +59,9 @@ def _sha256(s):
""" """
if isinstance(s, six.string_types): if isinstance(s, six.string_types):
try: try:
s = six.u(s).encode('utf-8') s = six.u(s).encode("utf-8")
except Exception: except Exception:
s = s.encode('utf-8') s = s.encode("utf-8")
return hashlib.sha256(s).hexdigest() return hashlib.sha256(s).hexdigest()
...@@ -71,14 +71,14 @@ def _sha256(s): ...@@ -71,14 +71,14 @@ def _sha256(s):
def _stringify(dictionary): def _stringify(dictionary):
names = sorted(dictionary.keys()) names = sorted(dictionary.keys())
converted_dictionary = '{' converted_dictionary = "{"
for name in names: for name in names:
converted_dictionary += '"%s":%s,' % (name, str(dictionary[name])) converted_dictionary += '"%s":%s,' % (name, str(dictionary[name]))
if len(converted_dictionary) > 1: if len(converted_dictionary) > 1:
converted_dictionary = converted_dictionary[:-1] converted_dictionary = converted_dictionary[:-1]
converted_dictionary += '}' converted_dictionary += "}"
return converted_dictionary return converted_dictionary
...@@ -87,13 +87,13 @@ def _stringify(dictionary): ...@@ -87,13 +87,13 @@ def _stringify(dictionary):
def _compact(text): def _compact(text):
return text.replace(' ', '').replace('\n', '') return text.replace(" ", "").replace("\n", "")
# ---------------------------------------------------------- # ----------------------------------------------------------
def toPath(hash, suffix='.data'): def toPath(hash, suffix=".data"):
""" Returns the path on disk which corresponds to the hash given. """ Returns the path on disk which corresponds to the hash given.
Parameters: Parameters:
...@@ -159,7 +159,7 @@ def hashJSON(contents, description): ...@@ -159,7 +159,7 @@ def hashJSON(contents, description):
contents = copy.deepcopy(contents) # temporary copy contents = copy.deepcopy(contents) # temporary copy
del contents[description] del contents[description]
contents = simplejson.dumps(contents, sort_keys=True) contents = simplejson.dumps(contents, sort_keys=True)
return hashlib.sha256(contents.encode('utf-8')).hexdigest() return hashlib.sha256(contents.encode("utf-8")).hexdigest()
# ---------------------------------------------------------- # ----------------------------------------------------------
...@@ -175,11 +175,14 @@ def hashJSONFile(path, description): ...@@ -175,11 +175,14 @@ def hashJSONFile(path, description):
""" """
try: try:
with open(path, 'rb') as f: with open(path, "rb") as f:
# preserve order # preserve order
return hashJSON(simplejson.loads(f.read().decode('utf-8'), return hashJSON(
object_pairs_hook=collections.OrderedDict), simplejson.loads(
description) f.read().decode("utf-8"), object_pairs_hook=collections.OrderedDict
),
description,
)
except simplejson.JSONDecodeError: except simplejson.JSONDecodeError:
# falls back to normal file content hashing # falls back to normal file content hashing
return hashFileContents(path) return hashFileContents(path)
...@@ -195,9 +198,9 @@ def hashFileContents(path): ...@@ -195,9 +198,9 @@ def hashFileContents(path):
str: hash str: hash
""" """
with open(path, 'rb') as f: with open(path, "rb") as f:
sha256 = hashlib.sha256() sha256 = hashlib.sha256()
for chunk in iter(lambda: f.read(sha256.block_size * 1000), b''): for chunk in iter(lambda: f.read(sha256.block_size * 1000), b""):
sha256.update(chunk) sha256.update(chunk)
return sha256.hexdigest() return sha256.hexdigest()
...@@ -218,9 +221,14 @@ def hashDataset(database_name, protocol_name, set_name): ...@@ -218,9 +221,14 @@ def hashDataset(database_name, protocol_name, set_name):
str: hash str: hash
""" """
s = _compact("""{ s = (
_compact(
"""{
"database": "%s", "database": "%s",
"protocol": "%s", "protocol": "%s",
"set": "%s" "set": "%s"
}""") % (database_name, protocol_name, set_name) }"""
)
% (database_name, protocol_name, set_name)
)
return hash(s) return hash(s)
...@@ -42,24 +42,24 @@ helpers ...@@ -42,24 +42,24 @@ helpers
This module implements various helper methods and classes This module implements various helper methods and classes
""" """
import os
import errno import errno
import logging import logging
import os
from .algorithm import Algorithm
from .data import CachedDataSink
from .data import CachedDataSource from .data import CachedDataSource
from .data import RemoteDataSource from .data import RemoteDataSource
from .data import CachedDataSink
from .data import getAllFilenames from .data import getAllFilenames
from .data_loaders import DataLoaderList
from .data_loaders import DataLoader from .data_loaders import DataLoader
from .inputs import InputList from .data_loaders import DataLoaderList
from .inputs import Input from .inputs import Input
from .inputs import InputGroup from .inputs import InputGroup
from .outputs import SynchronizationListener from .inputs import InputList
from .outputs import OutputList
from .outputs import Output from .outputs import Output
from .outputs import OutputList
from .outputs import RemotelySyncedOutput from .outputs import RemotelySyncedOutput
from .algorithm import Algorithm from .outputs import SynchronizationListener
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -46,7 +46,6 @@ from functools import reduce ...@@ -46,7 +46,6 @@ from functools import reduce
import six import six
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -43,12 +43,12 @@ Validation for libraries ...@@ -43,12 +43,12 @@ Validation for libraries
""" """
import os import os
import simplejson as json import simplejson as json
from . import loader from . import loader
from . import utils from . import utils
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -44,10 +44,10 @@ executor. Safe in this context means that if the method raises an ...@@ -44,10 +44,10 @@ executor. Safe in this context means that if the method raises an
exception, it will catch it and return in a suitable form to the caller. exception, it will catch it and return in a suitable form to the caller.
""" """
import sys
import six
import imp import imp
import sys
import six
# ---------------------------------------------------------- # ----------------------------------------------------------
...@@ -79,11 +79,11 @@ def load_module(name, path, uses): ...@@ -79,11 +79,11 @@ def load_module(name, path, uses):
# loads used modules # loads used modules
for k, v in uses.items(): for k, v in uses.items():
retval.__dict__[k] = load_module(k, v['path'], v['uses']) retval.__dict__[k] = load_module(k, v["path"], v["uses"])
# execute the module code on the context of previously import modules # execute the module code on the context of previously import modules
with open(path, "rb") as f: with open(path, "rb") as f:
exec(compile(f.read(), path, 'exec'), retval.__dict__) exec(compile(f.read(), path, "exec"), retval.__dict__) # nosec
return retval return retval
...@@ -118,7 +118,7 @@ def run(obj, method, exc=None, *args, **kwargs): ...@@ -118,7 +118,7 @@ def run(obj, method, exc=None, *args, **kwargs):
""" """
try: try:
if method == '__new__': if method == "__new__":
return obj(*args, **kwargs) return obj(*args, **kwargs)
return getattr(obj, method)(*args, **kwargs) return getattr(obj, method)(*args, **kwargs)
......
...@@ -44,10 +44,8 @@ Validation of database protocol templates ...@@ -44,10 +44,8 @@ Validation of database protocol templates
import simplejson as json import simplejson as json
from .dataformat import DataFormat
from . import utils from . import utils
from .dataformat import DataFormat
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -89,17 +89,14 @@ def io_statistics(configuration, input_list=None, output_list=None): ...@@ -89,17 +89,14 @@ def io_statistics(configuration, input_list=None, output_list=None):
write_time += duration write_time += duration
blocks_written += output.nb_data_blocks_written blocks_written += output.nb_data_blocks_written
if 'result' in configuration: if "result" in configuration:
hash = configuration['result']['path'].replace('/', '') hash = configuration["result"]["path"].replace("/", "")
else: else:
hash = configuration['outputs'][output.name]['path'].replace( hash = configuration["outputs"][output.name]["path"].replace("/", "")
'/', '')
files.append(dict( files.append(
hash=hash, dict(hash=hash, size=size, blocks=output.nb_data_blocks_written,)
size=size, )
blocks=output.nb_data_blocks_written,
))
# Result # Result
return dict( return dict(
...@@ -128,13 +125,13 @@ def update(statistics, additional_statistics): ...@@ -128,13 +125,13 @@ def update(statistics, additional_statistics):
""" """
for k in statistics.keys(): for k in statistics.keys():
if k == 'files': if k == "files":
continue continue
for k2 in statistics[k].keys(): for k2 in statistics[k].keys():
statistics[k][k2] += additional_statistics[k][k2] statistics[k][k2] += additional_statistics[k][k2]
if 'files' in statistics: if "files" in statistics:
statistics['files'].extend(additional_statistics.get('files', [])) statistics["files"].extend(additional_statistics.get("files", []))
else: else:
statistics['files'] = additional_statistics.get('files', []) statistics["files"] = additional_statistics.get("files", [])
...@@ -42,16 +42,16 @@ utils ...@@ -42,16 +42,16 @@ utils
This module implements helper classes and functions. This module implements helper classes and functions.
""" """
import collections
import os import os
import shutil import shutil
import collections
import numpy import numpy
import simplejson import simplejson
import six import six
from . import hash from . import hash
# ---------------------------------------------------------- # ----------------------------------------------------------
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment