Commit a1e373d4 authored by André Anjos's avatar André Anjos 💬

Merge branch 'cleanup' into 'master'

Code cleanup

See merge request !62
parents c2f47ffd 7f188124
Pipeline #28262 passed with stage
in 29 minutes and 4 seconds
......@@ -6,12 +6,14 @@ repos:
hooks:
- id: black
language_version: python3.6
exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- id: check-added-large-files
- id: check-docstring-first
- id: flake8
......@@ -21,7 +23,7 @@ repos:
rev: 'master' # Update me!
hooks:
- id: bandit
exclude: beat/editor/test
exclude: beat/editor/test|beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: local
hooks:
- id: sphinx-build
......
......@@ -66,7 +66,9 @@ from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
def load_algorithm_prototype(prefix):
prototype_data = pkg_resources.resource_string(__name__, "prototypes/algorithm.json")
prototype_data = pkg_resources.resource_string(
__name__, "prototypes/algorithm.json"
)
if sys.version_info < (3, 6):
prototype_data = prototype_data.decode("utf-8")
......
......@@ -42,4 +42,4 @@ baseformat
Froward imports from :py:mod:`beat.backend.python.baseformat`
"""
from beat.backend.python.baseformat import *
from beat.backend.python.baseformat import * # noqa
......@@ -54,15 +54,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
:py:func:`beat.backend.python.data.foundSplitRanges`
"""
from beat.backend.python.data import mixDataIndices
from beat.backend.python.data import getAllFilenames
from beat.backend.python.data import DataSource
from beat.backend.python.data import CachedDataSource
from beat.backend.python.data import DatabaseOutputDataSource
from beat.backend.python.data import RemoteDataSource
from beat.backend.python.data import DataSink
from beat.backend.python.data import CachedDataSink
from beat.backend.python.data import StdoutDataSink
from beat.backend.python.data import load_data_index
from beat.backend.python.data import load_data_index_db
from beat.backend.python.data import foundSplitRanges
from beat.backend.python.data import mixDataIndices # noqa
from beat.backend.python.data import getAllFilenames # noqa
from beat.backend.python.data import DataSource # noqa
from beat.backend.python.data import CachedDataSource # noqa
from beat.backend.python.data import DatabaseOutputDataSource # noqa
from beat.backend.python.data import RemoteDataSource # noqa
from beat.backend.python.data import DataSink # noqa
from beat.backend.python.data import CachedDataSink # noqa
from beat.backend.python.data import StdoutDataSink # noqa
from beat.backend.python.data import load_data_index # noqa
from beat.backend.python.data import load_data_index_db # noqa
from beat.backend.python.data import foundSplitRanges # noqa
......@@ -46,6 +46,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
"""
from beat.backend.python.data_loaders import DataLoaderList
from beat.backend.python.data_loaders import DataLoader
from beat.backend.python.data_loaders import DataView
from beat.backend.python.data_loaders import DataLoaderList # noqa
from beat.backend.python.data_loaders import DataLoader # noqa
from beat.backend.python.data_loaders import DataView # noqa
......@@ -45,18 +45,14 @@ Forward importing from :py:mod:`beat.backend.python.dataformat`:
:py:class:`beat.backend.python.dataformat.Storage`
"""
import os
import copy
import six
import simplejson
from . import schema
from . import prototypes
from . import utils
from .baseformat import baseformat
from beat.backend.python.dataformat import Storage
from beat.backend.python.dataformat import Storage # noqa
from beat.backend.python.dataformat import DataFormat as BackendDataFormat
......@@ -132,7 +128,7 @@ class DataFormat(BackendDataFormat):
if data is None: # loads prototype and validates it
self.data, self.errors = prototypes.load("dataformat")
assert not self.errors, "\n * %s" % "\n *".join(self.errors)
assert not self.errors, "\n * %s" % "\n *".join(self.errors) # nosec
else:
......
......@@ -49,7 +49,7 @@ import socket
import tempfile
import time
import docker
import subprocess as sp
import subprocess as sp # nosec
import logging
from beat.core import stats
......@@ -140,14 +140,14 @@ class Host(object):
self.processing_environments.keys(),
)
)[0]
except:
except IndexError:
try:
return list(
filter(
lambda x: x.startswith(name + " ("), self.db_environments.keys()
)
)[0]
except:
except IndexError:
return None
@property
......@@ -245,10 +245,10 @@ class Host(object):
replacement = True
else:
keep = True
except:
except Exception:
replacement = True
except:
except Exception:
keep = True
elif new_version is not None:
......@@ -493,7 +493,7 @@ class Host(object):
try:
return simplejson.loads(stdout)[0]["State"]["Status"]
except:
except Exception:
return None
def logs(self, container):
......@@ -569,7 +569,7 @@ class Host(object):
status = self.wait(container)
output = self.logs(container)
except Exception as e:
except Exception:
return 1, None
finally:
......
......@@ -79,7 +79,7 @@ def enumerate_packages(host, environment_name):
package_lines = output.split("\n")
for package_line in package_lines:
information = re.split("\s+", package_line)
information = re.split(r"\s+", package_line)
if len(information) == 4:
packages.append({"name": information[0], "version": information[1]})
......
......@@ -33,7 +33,8 @@
# #
###################################################################################
# flake8: noqa
from .docker import DockerExecutor
from .local import LocalExecutor
from .subprocess import SubprocessExecutor
from .subprocess import SubprocessExecutor # nosec
......@@ -44,14 +44,9 @@ Execution utilities
import os
import sys
import glob
import collections
import logging
logger = logging.getLogger(__name__)
import simplejson
from .. import schema
......@@ -62,6 +57,9 @@ from .. import stats
from beat.backend.python.helpers import convert_experiment_configuration_to_container
logger = logging.getLogger(__name__)
class BaseExecutor(object):
"""Executors runs the code given an execution block information
......
......@@ -47,8 +47,6 @@ import shutil
import logging
import requests
import simplejson
import zmq
import docker
from beat.backend.python.execution import MessageHandler
......@@ -203,7 +201,7 @@ class DockerExecutor(RemoteExecutor):
# Determine the docker image to use for the databases
try:
databases_environment = self.host.db2docker(database_paths.keys())
except:
except Exception:
raise RuntimeError(
"No environment found for the databases `%s' "
"- available environments are %s"
......
......@@ -49,24 +49,17 @@ import tempfile
import shutil
import zmq
import time
import logging
logger = logging.getLogger(__name__)
import simplejson
from .base import BaseExecutor
from beat.backend.python.helpers import create_inputs_from_configuration
from beat.backend.python.helpers import create_outputs_from_configuration
from beat.backend.python.helpers import AccessMode
from beat.backend.python.execution import AlgorithmExecutor
from beat.backend.python.execution import MessageHandler
from beat.backend.python.execution import LoopExecutor
from beat.backend.python.execution import LoopMessageHandler
from .base import BaseExecutor
logger = logging.getLogger(__name__)
class LocalExecutor(BaseExecutor):
"""LocalExecutor runs the code given an execution block information
......
......@@ -42,17 +42,16 @@ remote
Execution utilities
"""
import zmq
import logging
logger = logging.getLogger(__name__)
from .base import BaseExecutor
from beat.backend.python.helpers import create_inputs_from_configuration
from beat.backend.python.helpers import create_outputs_from_configuration
from beat.backend.python.helpers import AccessMode
logger = logging.getLogger(__name__)
class RemoteExecutor(BaseExecutor):
......
......@@ -41,16 +41,12 @@ subprocess
Execution utilities
"""
from __future__ import absolute_import
import os
import shutil
import logging
import sys
import subprocess as sp
import subprocess as sp # nosec
import tempfile
import requests
import zmq
from beat.backend.python.execution import MessageHandler
......@@ -415,7 +411,7 @@ class SubprocessExecutor(RemoteExecutor):
algorithm_process.kill()
status = algorithm_process.wait()
except:
except Exception:
algorithm_process.kill()
status = algorithm_process.wait()
......
......@@ -231,9 +231,9 @@ class Experiment(object):
if data is None: # loads prototype and validates it
experiment_data, self.errors = prototypes.load("experiment")
assert not self.errors, "\n * %s" % "\n *".join(self.errors)
assert not self.errors, "\n * %s" % "\n *".join(self.errors) # nosec
toolchain_data, self.errors = prototypes.load("toolchain")
assert not self.errors, "\n * %s" % "\n *".join(self.errors)
assert not self.errors, "\n * %s" % "\n *".join(self.errors) # nosec
elif isinstance(data, (tuple, list)): # the user has passed a tuple
......@@ -477,7 +477,7 @@ class Experiment(object):
self.errors.append(
"/loop/%s/nb_slots: you have set the number "
"of slots for algorithm `%s' to %d, but it is not "
"splittable" % (analyzername, thisalgo.name, loop["nb_slots"])
"splittable" % (algoname, thisalgo.name, loop["nb_slots"])
)
# check parameter consistence
......@@ -740,7 +740,7 @@ class Experiment(object):
from_dtype = self.algorithms[block["algorithm"]].output_map[algout]
from_name = "block"
else:
self.errors.append("Unknown endpoint %s" % to_endpt[0])
self.errors.append("Unknown endpoint %s" % from_endpt[0])
continue
to_endpt = connection["to"].split(".", 1)
......@@ -869,7 +869,7 @@ class Experiment(object):
# for the grouping properties for the inputs
# create channel groups
chain_in = collections.Counter(input_connections)
chain_groups = collections.Counter(input_connections)
# now check the algorithm for conformance
algo_groups = self.algorithms[self.analyzers[name]["algorithm"]].groups
......@@ -926,7 +926,7 @@ class Experiment(object):
)
# makes sure we don't have multiple incomming connections
assert len(_connections) == len(connections), (
assert len(_connections) == len(connections), ( # nosec
"detected multiple input "
"connections for block `%s' on experiment `%s'" % (name, self.label)
)
......@@ -975,7 +975,6 @@ class Experiment(object):
# then go one by one generating the input **and** output hashes
# until all is done.
block_config = self.blocks[block]
retval[algo_endpt] = {
"from": "%s.%s" % (block, output),
"channel": channel,
......@@ -1181,8 +1180,6 @@ class Experiment(object):
def dot_diagram(self):
"""Returns a dot diagram representation of the experiment"""
from .drawing import create_port_table
title = "Experiment: %s" % self.label
def __label_callback(type, name):
......
......@@ -48,8 +48,8 @@ import collections
import simplejson
from beat.backend.python.hash import *
from beat.backend.python.hash import _sha256
from beat.backend.python.hash import * # noqa
from beat.backend.python.hash import _sha256 # noqa
from beat.backend.python.hash import _stringify
from beat.backend.python.hash import _compact
......@@ -164,7 +164,7 @@ def hashJSONStr(contents, description):
"""
try:
return hashJSON(
return hashJSON( # noqa
simplejson.loads(contents, object_pairs_hook=collections.OrderedDict),
description,
) # preserve order
......
......@@ -47,6 +47,6 @@ Forward imported from :py:mod:`beat.backend.python.inputs`:
"""
from beat.backend.python.inputs import InputList
from beat.backend.python.inputs import Input
from beat.backend.python.inputs import InputGroup
from beat.backend.python.inputs import InputList # noqa
from beat.backend.python.inputs import Input # noqa
from beat.backend.python.inputs import InputGroup # noqa
......@@ -147,7 +147,7 @@ class Library(BackendLibrary):
# At this point, `data' can be a dictionary or ``None``
if data is None: # loads the default declaration for an library
self.data, self.errors = prototypes.load("library")
assert not self.errors, "\n * %s" % "\n *".join(self.errors)
assert not self.errors, "\n * %s" % "\n *".join(self.errors) # nosec
else: # just assign it
# this runs basic validation, including JSON loading if required
self.data, self.errors = schema.validate("library", data)
......
......@@ -43,4 +43,4 @@ Forward imports from :py:mod:`beat.backend.python.loader`
"""
from beat.backend.python.loader import *
from beat.backend.python.loader import * # noqa
......@@ -46,6 +46,6 @@ Forward imported from :py:mod:`beat.backend.python.outputs`:
"""
from beat.backend.python.outputs import SynchronizationListener
from beat.backend.python.outputs import Output
from beat.backend.python.outputs import OutputList
from beat.backend.python.outputs import SynchronizationListener # noqa
from beat.backend.python.outputs import Output # noqa
from beat.backend.python.outputs import OutputList # noqa
......@@ -44,6 +44,7 @@ Validation for plotters
import os
import six
import sys
from . import dataformat
from . import algorithm
......@@ -227,7 +228,7 @@ class Plotter(object):
# At this point, `data' can be a dictionary or ``None``
if data is None: # loads the default declaration for an algorithm
self.data, self.errors = prototypes.load("plotter")
assert not self.errors, "\n * %s" % "\n *".join(self.errors)
assert not self.errors, "\n * %s" % "\n *".join(self.errors) # nosec
else: # just assign it
# this runs basic validation, including JSON loading if required
self.data, self.errors = schema.validate("plotter", data)
......@@ -400,7 +401,7 @@ class Plotter(object):
self.uses_dict(),
),
)
except Exception as e:
except Exception:
if exc is not None:
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
......
......@@ -37,8 +37,6 @@
import pkg_resources
import simplejson
from .. import schema
......
class Algorithm:
def process(self, inputs, data_loaders, outputs):
# TODO: Implement this algorithm
......
class Plotter:
def process(self, input):
# TODO: Implement this plotter
#return image as a string
# return image as a string
pass
class View:
def setup(
self,
root_folder,
outputs,
parameters,
force_start_index=None,
force_end_index=None,
):
"""Initializes the database"""
def setup(self, root_folder, outputs, parameters, force_start_index=None, force_end_index=None):
'''Initializes the database'''
return True
return True
def done(self):
"""Should return ``True``, when data is finished"""
return True
def done(self):
'''Should return ``True``, when data is finished'''
def next(self):
"""Loads the next data block on ``outputs``"""
return True
def next(self):
'''Loads the next data block on ``outputs``'''
return True
return True
......@@ -76,4 +76,4 @@
},
"required": ["type", "api_version", "schema_version"]
}
\ No newline at end of file
}
......@@ -160,4 +160,4 @@
},
"required": ["type", "api_version", "schema_version"]
}
\ No newline at end of file
}
......@@ -220,4 +220,4 @@
}
}
}
}
\ No newline at end of file
}
......@@ -2,7 +2,6 @@ from ..agent import MessageHandler
from ..dataformat import DataFormat
from ..data import DataSource
from ..inputs import RemoteInput
from ..inputs import RemoteException
from ..inputs import Input
from ..inputs import InputGroup
from ..inputs import InputList
......@@ -12,18 +11,16 @@ import time
import zmq
prefix = pkg_resources.resource_filename(__name__, '../test/prefix')
prefix = pkg_resources.resource_filename(__name__, "../test/prefix")
print(prefix)
class CustomDataSource(DataSource):
def __init__(self, nb_data_units, dataformat):
self.nb_data_units = nb_data_units
self.current = 0
self.dataformat = dataformat
self.file = open('benchmark.data', 'rb')
self.file = open("benchmark.data", "rb")
self.unpack = True
def next(self):
......@@ -50,20 +47,20 @@ class CustomDataSource(DataSource):
def main():
dataformat = DataFormat(prefix, 'user/empty_1d_array_of_integers/1')
dataformat = DataFormat(prefix, "user/empty_1d_array_of_integers/1")
data = dataformat.type(value=np.random.randint(100000, size=(1000000,), dtype=np.int32))
data = dataformat.type(
value=np.random.randint(100000, size=(1000000,), dtype=np.int32)
)
with open('benchmark.data', 'wb') as f:
with open("benchmark.data", "wb") as f:
f.write(data.pack())
#-----------------------
# -----------------------
data_source = CustomDataSource(10000, dataformat)
print('Nb Data units: %d' % data_source.nb_data_units)
print("Nb Data units: %d" % data_source.nb_data_units)
t1 = time.time()
......@@ -72,12 +69,14 @@ def main():
t2 = time.time()
print('Datasource (unpack): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
print(
"Datasource (unpack): %.3fs (%.3fms/unit)"
% (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
)
data_source.reset()
#-----------------------
# -----------------------
data_source.unpack = False
......@@ -88,16 +87,16 @@ def main():
t2 = time.time()
print('Datasource (packed): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
print(
"Datasource (packed): %.3fs (%.3fms/unit)"
% (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
)
data_source.reset()
# -----------------------
#-----------------------
input = Input('a', 'user/empty_1d_array_of_integers/1', data_source)
input = Input("a", "user/empty_1d_array_of_integers/1", data_source)
t1 = time.time()
......@@ -106,13 +105,14 @@ def main():
t2 = time.time()
print('Input (unpack): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
print(
"Input (unpack): %.3fs (%.3fms/unit)"
% (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
)
data_source.reset()
#-----------------------
# -----------------------
data_source.unpack = False
......@@ -123,37 +123,36 @@ def main():
t2 = time.time()
print('Input (packed): %.3fs (%.3fms/unit)' % (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units))
print(
"Input (packed): %.3fs (%.3fms/unit)"
% (t2 - t1, (t2 - t1) * 1000.0 / data_source.nb_data_units)
)
data_source.reset()
# -----------------------
#-----------------------
group = InputGroup('channel')
group = InputGroup("channel")
group.add(input)
input_list = InputList()
input_list.add(group)
server_context = zmq.Context()
server_socket = server_context.socket(zmq.PAIR)
address = 'tcp://127.0.0.1'
address = "tcp://127.0.0.1"
port = server_socket.bind_to_random_port(address)
address += ':%d' % port
address += ":%d" % port
message_handler = MessageHandler(input_list, server_context, server_socket)
client_context = zmq.Context()
client_socket = client_context.socket(zmq.PAIR)
client_socket.connect(address)