Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
7ab0f93a
Commit
7ab0f93a
authored
Apr 13, 2021
by
Flavio TARSETTI
Browse files
Merge branch 'pyproject_toml' into 'master'
Add pyproject.toml See merge request
!137
parents
f84d0710
0a0ccef1
Pipeline
#49652
failed with stages
in 174 minutes and 51 seconds
Changes
34
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
.isort.cfg
View file @
7ab0f93a
...
...
@@ -6,3 +6,5 @@ use_parentheses=true
ensure_newline_before_comments=true
line_length=88
force_single_line=true
order_by_type=true
lines_between_types=1
.pre-commit-config.yaml
View file @
7ab0f93a
...
...
@@ -2,17 +2,17 @@
# See https://pre-commit.com/hooks.html for more hooks
repos
:
-
repo
:
https://github.com/timothycrosley/isort
rev
:
5.
0.4
rev
:
5.
8.0
hooks
:
-
id
:
isort
files
:
.*.py
-
repo
:
https://github.com/psf/black
rev
:
19.10b0
rev
:
20.8b1
hooks
:
-
id
:
black
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
repo
:
https://github.com/pre-commit/pre-commit-hooks
rev
:
v3.
1
.0
rev
:
v3.
4
.0
hooks
:
-
id
:
check-ast
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
...
...
@@ -26,11 +26,11 @@ repos:
-
id
:
check-yaml
exclude
:
conda/meta.yaml
-
repo
:
https://github.com/PyCQA/flake8
rev
:
3.
8.3
rev
:
3.
9.0
hooks
:
-
id
:
flake8
-
repo
:
https://github.com/PyCQA/bandit
rev
:
1.
6.2
rev
:
1.
7.0
hooks
:
-
id
:
bandit
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
...
...
beat/core/algorithm.py
View file @
7ab0f93a
...
...
@@ -280,8 +280,7 @@ class Algorithm(BackendAlgorithm):
self
.
_check_language_consistence
()
def
_check_endpoint_uniqueness
(
self
):
"""Checks for name clashes accross input/output groups
"""
"""Checks for name clashes accross input/output groups"""
all_input_names
=
[]
for
group
in
self
.
groups
:
...
...
@@ -325,8 +324,7 @@ class Algorithm(BackendAlgorithm):
self
.
_validate_format
(
type_name
,
group_name
,
name
,
thisformat
)
def
_validate_required_dataformats
(
self
,
dataformat_cache
):
"""Makes sure we can load all requested formats
"""
"""Makes sure we can load all requested formats"""
for
group
in
self
.
groups
:
...
...
beat/core/bcp/worker.py
View file @
7ab0f93a
...
...
@@ -65,10 +65,12 @@ import os
import
signal
import
sys
import
tempfile
from
socket
import
gethostname
import
simplejson
as
json
import
zmq
from
docopt
import
docopt
from
..bcpapi
import
BCP
...
...
beat/core/bcpapi/broker.py
View file @
7ab0f93a
...
...
@@ -40,6 +40,7 @@ Inspired by the Majordomo Protocol Broker
import
logging
import
signal
import
time
from
binascii
import
hexlify
import
zmq
...
...
beat/core/bcpapi/client.py
View file @
7ab0f93a
...
...
@@ -73,8 +73,7 @@ class BeatComputationClient(object):
logger
.
info
(
"I: connecting to broker at %s..."
,
self
.
broker
)
def
send
(
self
,
service
,
request
):
"""Send request to broker
"""
"""Send request to broker"""
if
not
isinstance
(
request
,
list
):
request
=
[
request
]
...
...
beat/core/bcpapi/processor.py
View file @
7ab0f93a
...
...
@@ -44,8 +44,7 @@ logger = logging.getLogger(__name__)
class
BeatComputationProcessor
(
object
):
"""BEAT Computation Protocol Processor API, Python version
"""
"""BEAT Computation Protocol Processor API, Python version"""
def
__init__
(
self
,
poller
,
address
,
verbose
=
False
):
self
.
verbose
=
verbose
...
...
beat/core/bcpapi/worker.py
View file @
7ab0f93a
...
...
@@ -46,8 +46,7 @@ logger = logging.getLogger(__name__)
class
BeatComputationWorker
(
object
):
"""BEAT Computation Protocol Worker API, Python version
"""
"""BEAT Computation Protocol Worker API, Python version"""
HEARTBEAT_LIVENESS
=
3
# 3-5 is reasonable
...
...
beat/core/dock.py
View file @
7ab0f93a
...
...
@@ -51,6 +51,7 @@ import time
import
docker
import
simplejson
as
json
from
packaging
import
version
from
beat.core
import
stats
...
...
@@ -61,8 +62,7 @@ logger = logging.getLogger(__name__)
class
Host
(
object
):
"""An object of this class can connect to the docker host and resolve stuff
"""
"""An object of this class can connect to the docker host and resolve stuff"""
images_cache
=
{}
...
...
@@ -807,19 +807,17 @@ class Container:
self
.
_temporary_filesystems
[
path
]
=
size
def
set_name
(
self
,
name
):
"""
Set the name to be used by the container in place of the docker
"""Set the name to be used by the container in place of the docker
auto generated one.
"""
self
.
_name
=
name
def
set_workdir
(
self
,
workdir
):
""" Set the work folder to be used by the container
"""
"""Set the work folder to be used by the container"""
self
.
_workdir
=
workdir
def
set_entrypoint
(
self
,
entrypoint
):
""" Set the entry point to be used by the container
"""
"""Set the entry point to be used by the container"""
self
.
_entrypoint
=
entrypoint
def
add_volume
(
self
,
path
,
mount_path
,
read_only
=
True
):
...
...
@@ -952,8 +950,7 @@ class Container:
@
property
def
environment_variables
(
self
):
"""Returns the environment variables to set on this container.
"""
"""Returns the environment variables to set on this container."""
environment_variables
=
[]
for
k
,
v
in
self
.
_environment_variables
.
items
():
...
...
beat/core/execution/base.py
View file @
7ab0f93a
...
...
@@ -302,8 +302,7 @@ class BaseExecutor(object):
return
self
def
__exit__
(
self
,
exc_type
,
exc_value
,
traceback
):
"""Closes all sinks and disconnects inputs and outputs
"""
"""Closes all sinks and disconnects inputs and outputs"""
for
sink
in
self
.
data_sinks
:
# we save the output only if no valid error has been thrown
...
...
beat/core/execution/docker.py
View file @
7ab0f93a
...
...
@@ -44,6 +44,7 @@ Execution utilities
import
logging
import
os
import
shutil
from
collections
import
namedtuple
import
requests
...
...
@@ -191,7 +192,10 @@ class DockerExecutor(RemoteExecutor):
databases_infos
=
{}
for
db_name
,
db_object
,
in
self
.
databases
.
items
():
for
(
db_name
,
db_object
,
)
in
self
.
databases
.
items
():
json_path
=
os
.
path
.
join
(
root_folder
,
db_name
+
".json"
)
with
open
(
json_path
,
"r"
)
as
f
:
...
...
beat/core/hash.py
View file @
7ab0f93a
...
...
@@ -86,23 +86,20 @@ def hashBlockOutput(
# Note: 'block_name' and 'algorithm_name' aren't used to compute the hash,
# but are useful when an application wants to implement its own hash
# function
s
=
(
_compact
(
"""{
s
=
_compact
(
"""{
"algorithm": "%s",
"parameters": %s,
"environment": %s,
"inputs": %s,
"output": "%s"
}"""
)
%
(
algorithm_hash
,
_stringify
(
parameters
),
_stringify
(
environment
),
_stringify
(
input_hashes
),
output_name
,
)
)
%
(
algorithm_hash
,
_stringify
(
parameters
),
_stringify
(
environment
),
_stringify
(
input_hashes
),
output_name
,
)
return
hash
(
s
)
...
...
@@ -131,23 +128,20 @@ def hashAnalyzer(
# Note: 'analyzer_name' isn't used to compute the hash, but is useful when
# an applications want to implement its own hash function
s
=
(
_compact
(
"""{
s
=
_compact
(
"""{
"algorithm_name": "%s",
"algorithm": "%s",
"parameters": %s,
"environment": %s,
"inputs": %s
}"""
)
%
(
algorithm_name
,
algorithm_hash
,
_stringify
(
parameters
),
_stringify
(
environment
),
_stringify
(
input_hashes
),
)
)
%
(
algorithm_name
,
algorithm_hash
,
_stringify
(
parameters
),
_stringify
(
environment
),
_stringify
(
input_hashes
),
)
return
hash
(
s
)
...
...
beat/core/plotter.py
View file @
7ab0f93a
...
...
@@ -277,8 +277,7 @@ class Plotter(object):
_check_language_consistence
=
algorithm
.
Algorithm
.
_check_language_consistence
def
_validate_dataformat
(
self
,
dataformat_cache
):
"""Makes sure we can load the requested format
"""
"""Makes sure we can load the requested format"""
name
=
self
.
data
[
"dataformat"
]
...
...
@@ -304,8 +303,7 @@ class Plotter(object):
@
property
def
name
(
self
):
"""The name of this object
"""
"""The name of this object"""
return
self
.
_name
or
"__unnamed_plotter__"
@
name
.
setter
...
...
beat/core/prototypes/database.py
View file @
7ab0f93a
...
...
@@ -7,7 +7,12 @@ from beat.backend.python.database import View
class
FooView
(
View
):
def
setup
(
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
):
"""Initializes the database"""
...
...
beat/core/schema/__init__.py
View file @
7ab0f93a
...
...
@@ -67,25 +67,25 @@ def load_schema(schema_name, version=1):
"""Returns a JSON validator for the schema given the relative name
Parameters:
Parameters:
schema_name (str): the name of the schema to load. This value corresponds
to the filename inside our schema directory (where this file is located)
and should *exclude* the extension ``.json``.
schema_name (str): the name of the schema to load. This value corresponds
to the filename inside our schema directory (where this file is located)
and should *exclude* the extension ``.json``.
version (int): the version of the schema to use.
version (int): the version of the schema to use.
Returns:
Returns:
jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
jsonschema.Draft4Validator: An instance of a JSON schema draft-4 validator.
Raises:
Raises:
jsonschema.SchemaError: If there is an error on the schema.
jsonschema.SchemaError: If there is an error on the schema.
"""
"""
fname
=
pkg_resources
.
resource_filename
(
__name__
,
os
.
path
.
join
(
schema_name
,
"%d.json"
%
version
)
...
...
@@ -109,54 +109,54 @@ def load_schema(schema_name, version=1):
def
validate
(
schema_name
,
data
):
"""Validates the input data using the schema
This function handles schema versionning in the context of BEAT transparently
by first peeking the schema version required by the JSON data and then
validating the JSON data against the proper schema version for the respective
type.
This function handles schema versionning in the context of BEAT transparently
by first peeking the schema version required by the JSON data and then
validating the JSON data against the proper schema version for the respective
type.
Example:
Example:
.. code-block:: python
.. code-block:: python
try:
cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
except json.JSONDecodeError as e:
print(e)
try:
cleaned_data, error_list = validate('toolchain', '/to/my/file.json')
except json.JSONDecodeError as e:
print(e)
Parameters:
Parameters:
schema_name (str): The relative path to the schema to use for validation.
For example, to validate a toolchain, use ``'toolchain'``.
schema_name (str): The relative path to the schema to use for validation.
For example, to validate a toolchain, use ``'toolchain'``.
data (object, str, file): The piece of data to validate. The input can be a
valid python object that represents a JSON structure, a file, from which
the JSON contents will be read out or a string.
data (object, str, file): The piece of data to validate. The input can be a
valid python object that represents a JSON structure, a file, from which
the JSON contents will be read out or a string.
If ``data`` is a string and represents a valid filesystem path, the
relevant file will be opened and read as with
:py:func:`json.load``. Otherwise, it will be considered to be
string containing a valid JSON structure that will be loaded as with
:py:func:`json.loads`.
If ``data`` is a string and represents a valid filesystem path, the
relevant file will be opened and read as with
:py:func:`json.load``. Otherwise, it will be considered to be
string containing a valid JSON structure that will be loaded as with
:py:func:`json.loads`.
Note that if the file is opened and read internally using
:py:func:`json.load`, exceptions may be thrown by that subsystem,
concerning the file structure. Consult the manual page for
:py:mod:`simplejson` for details.
Note that if the file is opened and read internally using
:py:func:`json.load`, exceptions may be thrown by that subsystem,
concerning the file structure. Consult the manual page for
:py:mod:`simplejson` for details.
Returns:
Returns:
A tuple with two elements: the cleaned JSON data structure, after
processing and a list of errors found by ``jsonschema``. If no errors
occur, then returns an empty list for the second element of the tuple.
A tuple with two elements: the cleaned JSON data structure, after
processing and a list of errors found by ``jsonschema``. If no errors
occur, then returns an empty list for the second element of the tuple.
Raises:
Raises:
jsonschema.SchemaError: If there is an error on the schema.
jsonschema.SchemaError: If there is an error on the schema.
"""
"""
try
:
data
=
maybe_load_json
(
data
)
...
...
beat/core/scripts/migrate_db_to_v2.py
View file @
7ab0f93a
...
...
@@ -53,9 +53,11 @@ Options:
import
copy
import
os
import
sys
from
collections
import
OrderedDict
import
simplejson
as
json
from
docopt
import
docopt
from
..database
import
Database
...
...
beat/core/scripts/worker.py
View file @
7ab0f93a
...
...
@@ -72,6 +72,7 @@ except ImportError:
import
queue
as
Queue
import
tempfile
from
socket
import
gethostname
from
docopt
import
docopt
...
...
@@ -84,7 +85,7 @@ from ..version import __version__
from
..worker
import
WorkerController
stop
=
False
logger
=
None
logger
=
logging
.
getLogger
(
__name__
)
# ----------------------------------------------------------
...
...
@@ -250,9 +251,6 @@ def main(user_input=None):
beat_core_logger
.
setLevel
(
logging
.
WARNING
)
beat_backend_logger
.
setLevel
(
logging
.
WARNING
)
global
logger
logger
=
logging
.
getLogger
(
__name__
)
# Check the prefix path
prefix
=
args
[
"--prefix"
]
if
args
[
"--prefix"
]
is
not
None
else
"."
if
not
os
.
path
.
exists
(
prefix
):
...
...
beat/core/test/prefix/databases/simple/1.py
View file @
7ab0f93a
...
...
@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class
View
(
BaseView
):
def
setup
(
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
):
"""Initializes the database"""
...
...
beat/core/test/prefix/databases/simple/2.py
View file @
7ab0f93a
...
...
@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class
View
(
BaseView
):
def
setup
(
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
):
"""Initializes the database"""
...
...
beat/core/test/prefix/databases/simple_env/1.py
View file @
7ab0f93a
...
...
@@ -43,7 +43,12 @@ from beat.backend.python.database import View as BaseView
class
View
(
BaseView
):
def
setup
(
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
self
,
root_folder
,
outputs
,
parameters
,
start_index
=
None
,
end_index
=
None
,
):
"""Initializes the database"""
...
...
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment