Commit b8fd9960 authored by Samuel GAIST's avatar Samuel GAIST

[backend] Pre-commit cleanup

parent d1be9948
Pipeline #40902 passed with stage
in 5 minutes and 2 seconds
......@@ -42,20 +42,19 @@ algorithm
Validation for algorithms
"""
import logging
import os
import sys
import logging
import six
import numpy
import simplejson as json
import six
from . import dataformat
from . import library
from . import loader
from . import utils
logger = logging.getLogger(__name__)
......
This diff is collapsed.
......@@ -42,23 +42,24 @@ data
Data I/O classes and functions
"""
import abc
import glob
import logging
import os
import re
import glob
import simplejson as json
import time
import abc
import zmq
import logging
import six
from functools import reduce
from collections import namedtuple
from functools import reduce
import simplejson as json
import six
import zmq
from .hash import hashFileContents
from .dataformat import DataFormat
from .algorithm import Algorithm
from .dataformat import DataFormat
from .exceptions import RemoteException
from .hash import hashFileContents
logger = logging.getLogger(__name__)
......@@ -107,8 +108,8 @@ def mixDataIndices(list_of_data_indices):
for index in range(start, end + 1):
done = False
for l in list_of_data_indices:
for indices in l:
for data_indices in list_of_data_indices:
for indices in data_indices:
if indices[1] == index:
result.append((current_start, index))
current_start = index + 1
......
......@@ -46,7 +46,6 @@ import six
from .data import mixDataIndices
# ----------------------------------------------------------
......
......@@ -42,23 +42,22 @@ database
Validation of databases
"""
import itertools
import os
import sys
import six
import simplejson as json
import itertools
import numpy as np
from collections import namedtuple
import numpy as np
import simplejson as json
import six
from . import loader
from . import utils
from .protocoltemplate import ProtocolTemplate
from .dataformat import DataFormat
from .outputs import OutputList
from .exceptions import OutputError
from .outputs import OutputList
from .protocoltemplate import ProtocolTemplate
# ----------------------------------------------------------
......
......@@ -42,17 +42,16 @@ dataformat
Validation and parsing for dataformats
"""
import re
import copy
import re
import six
import numpy
import simplejson as json
import six
from . import utils
from .baseformat import baseformat
# ----------------------------------------------------------
......
......@@ -42,13 +42,13 @@ hash
Various functions for hashing platform contributions and others
"""
import hashlib
import simplejson
import collections
import copy
import six
import hashlib
import os
import simplejson
import six
# ----------------------------------------------------------
......@@ -59,9 +59,9 @@ def _sha256(s):
"""
if isinstance(s, six.string_types):
try:
s = six.u(s).encode('utf-8')
s = six.u(s).encode("utf-8")
except Exception:
s = s.encode('utf-8')
s = s.encode("utf-8")
return hashlib.sha256(s).hexdigest()
......@@ -71,14 +71,14 @@ def _sha256(s):
def _stringify(dictionary):
names = sorted(dictionary.keys())
converted_dictionary = '{'
converted_dictionary = "{"
for name in names:
converted_dictionary += '"%s":%s,' % (name, str(dictionary[name]))
if len(converted_dictionary) > 1:
converted_dictionary = converted_dictionary[:-1]
converted_dictionary += '}'
converted_dictionary += "}"
return converted_dictionary
......@@ -87,13 +87,13 @@ def _stringify(dictionary):
def _compact(text):
return text.replace(' ', '').replace('\n', '')
return text.replace(" ", "").replace("\n", "")
# ----------------------------------------------------------
def toPath(hash, suffix='.data'):
def toPath(hash, suffix=".data"):
""" Returns the path on disk which corresponds to the hash given.
Parameters:
......@@ -159,7 +159,7 @@ def hashJSON(contents, description):
contents = copy.deepcopy(contents) # temporary copy
del contents[description]
contents = simplejson.dumps(contents, sort_keys=True)
return hashlib.sha256(contents.encode('utf-8')).hexdigest()
return hashlib.sha256(contents.encode("utf-8")).hexdigest()
# ----------------------------------------------------------
......@@ -175,11 +175,14 @@ def hashJSONFile(path, description):
"""
try:
with open(path, 'rb') as f:
with open(path, "rb") as f:
# preserve order
return hashJSON(simplejson.loads(f.read().decode('utf-8'),
object_pairs_hook=collections.OrderedDict),
description)
return hashJSON(
simplejson.loads(
f.read().decode("utf-8"), object_pairs_hook=collections.OrderedDict
),
description,
)
except simplejson.JSONDecodeError:
# falls back to normal file content hashing
return hashFileContents(path)
......@@ -195,9 +198,9 @@ def hashFileContents(path):
str: hash
"""
with open(path, 'rb') as f:
with open(path, "rb") as f:
sha256 = hashlib.sha256()
for chunk in iter(lambda: f.read(sha256.block_size * 1000), b''):
for chunk in iter(lambda: f.read(sha256.block_size * 1000), b""):
sha256.update(chunk)
return sha256.hexdigest()
......@@ -218,9 +221,14 @@ def hashDataset(database_name, protocol_name, set_name):
str: hash
"""
s = _compact("""{
s = (
_compact(
"""{
"database": "%s",
"protocol": "%s",
"set": "%s"
}""") % (database_name, protocol_name, set_name)
}"""
)
% (database_name, protocol_name, set_name)
)
return hash(s)
......@@ -42,24 +42,24 @@ helpers
This module implements various helper methods and classes
"""
import os
import errno
import logging
import os
from .algorithm import Algorithm
from .data import CachedDataSink
from .data import CachedDataSource
from .data import RemoteDataSource
from .data import CachedDataSink
from .data import getAllFilenames
from .data_loaders import DataLoaderList
from .data_loaders import DataLoader
from .inputs import InputList
from .data_loaders import DataLoaderList
from .inputs import Input
from .inputs import InputGroup
from .outputs import SynchronizationListener
from .outputs import OutputList
from .inputs import InputList
from .outputs import Output
from .outputs import OutputList
from .outputs import RemotelySyncedOutput
from .algorithm import Algorithm
from .outputs import SynchronizationListener
logger = logging.getLogger(__name__)
......
......@@ -46,7 +46,6 @@ from functools import reduce
import six
# ----------------------------------------------------------
......
......@@ -43,12 +43,12 @@ Validation for libraries
"""
import os
import simplejson as json
from . import loader
from . import utils
# ----------------------------------------------------------
......
......@@ -44,10 +44,10 @@ executor. Safe in this context means that if the method raises an
exception, it will catch it and return in a suitable form to the caller.
"""
import sys
import six
import imp
import sys
import six
# ----------------------------------------------------------
......@@ -79,11 +79,11 @@ def load_module(name, path, uses):
# loads used modules
for k, v in uses.items():
retval.__dict__[k] = load_module(k, v['path'], v['uses'])
retval.__dict__[k] = load_module(k, v["path"], v["uses"])
# execute the module code on the context of previously import modules
with open(path, "rb") as f:
exec(compile(f.read(), path, 'exec'), retval.__dict__)
exec(compile(f.read(), path, "exec"), retval.__dict__) # nosec
return retval
......@@ -118,7 +118,7 @@ def run(obj, method, exc=None, *args, **kwargs):
"""
try:
if method == '__new__':
if method == "__new__":
return obj(*args, **kwargs)
return getattr(obj, method)(*args, **kwargs)
......
......@@ -44,10 +44,8 @@ Validation of database protocol templates
import simplejson as json
from .dataformat import DataFormat
from . import utils
from .dataformat import DataFormat
# ----------------------------------------------------------
......
......@@ -89,17 +89,14 @@ def io_statistics(configuration, input_list=None, output_list=None):
write_time += duration
blocks_written += output.nb_data_blocks_written
if 'result' in configuration:
hash = configuration['result']['path'].replace('/', '')
if "result" in configuration:
hash = configuration["result"]["path"].replace("/", "")
else:
hash = configuration['outputs'][output.name]['path'].replace(
'/', '')
hash = configuration["outputs"][output.name]["path"].replace("/", "")
files.append(dict(
hash=hash,
size=size,
blocks=output.nb_data_blocks_written,
))
files.append(
dict(hash=hash, size=size, blocks=output.nb_data_blocks_written,)
)
# Result
return dict(
......@@ -128,13 +125,13 @@ def update(statistics, additional_statistics):
"""
for k in statistics.keys():
if k == 'files':
if k == "files":
continue
for k2 in statistics[k].keys():
statistics[k][k2] += additional_statistics[k][k2]
if 'files' in statistics:
statistics['files'].extend(additional_statistics.get('files', []))
if "files" in statistics:
statistics["files"].extend(additional_statistics.get("files", []))
else:
statistics['files'] = additional_statistics.get('files', [])
statistics["files"] = additional_statistics.get("files", [])
......@@ -42,16 +42,16 @@ utils
This module implements helper classes and functions.
"""
import collections
import os
import shutil
import collections
import numpy
import simplejson
import six
from . import hash
# ----------------------------------------------------------
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment