Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
a1e373d4
Commit
a1e373d4
authored
Mar 18, 2019
by
André Anjos
💬
Browse files
Merge branch 'cleanup' into 'master'
Code cleanup See merge request
!62
parents
c2f47ffd
7f188124
Pipeline
#28262
passed with stage
in 29 minutes and 4 seconds
Changes
94
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
.pre-commit-config.yaml
View file @
a1e373d4
...
...
@@ -6,12 +6,14 @@ repos:
hooks
:
-
id
:
black
language_version
:
python3.6
exclude
:
beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
repo
:
https://github.com/pre-commit/pre-commit-hooks
rev
:
v2.0.0
hooks
:
-
id
:
trailing-whitespace
-
id
:
end-of-file-fixer
-
id
:
debug-statements
exclude
:
beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
id
:
check-added-large-files
-
id
:
check-docstring-first
-
id
:
flake8
...
...
@@ -21,7 +23,7 @@ repos:
rev
:
'
master'
# Update me!
hooks
:
-
id
:
bandit
exclude
:
beat/editor/test
exclude
:
beat/editor/test
|beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
repo
:
local
hooks
:
-
id
:
sphinx-build
...
...
beat/core/algorithm.py
View file @
a1e373d4
...
...
@@ -66,7 +66,9 @@ from beat.backend.python.algorithm import Algorithm as BackendAlgorithm
def
load_algorithm_prototype
(
prefix
):
prototype_data
=
pkg_resources
.
resource_string
(
__name__
,
"prototypes/algorithm.json"
)
prototype_data
=
pkg_resources
.
resource_string
(
__name__
,
"prototypes/algorithm.json"
)
if
sys
.
version_info
<
(
3
,
6
):
prototype_data
=
prototype_data
.
decode
(
"utf-8"
)
...
...
beat/core/baseformat.py
View file @
a1e373d4
...
...
@@ -42,4 +42,4 @@ baseformat
Froward imports from :py:mod:`beat.backend.python.baseformat`
"""
from
beat.backend.python.baseformat
import
*
from
beat.backend.python.baseformat
import
*
# noqa
beat/core/data.py
View file @
a1e373d4
...
...
@@ -54,15 +54,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
:py:func:`beat.backend.python.data.foundSplitRanges`
"""
from
beat.backend.python.data
import
mixDataIndices
from
beat.backend.python.data
import
getAllFilenames
from
beat.backend.python.data
import
DataSource
from
beat.backend.python.data
import
CachedDataSource
from
beat.backend.python.data
import
DatabaseOutputDataSource
from
beat.backend.python.data
import
RemoteDataSource
from
beat.backend.python.data
import
DataSink
from
beat.backend.python.data
import
CachedDataSink
from
beat.backend.python.data
import
StdoutDataSink
from
beat.backend.python.data
import
load_data_index
from
beat.backend.python.data
import
load_data_index_db
from
beat.backend.python.data
import
foundSplitRanges
from
beat.backend.python.data
import
mixDataIndices
# noqa
from
beat.backend.python.data
import
getAllFilenames
# noqa
from
beat.backend.python.data
import
DataSource
# noqa
from
beat.backend.python.data
import
CachedDataSource
# noqa
from
beat.backend.python.data
import
DatabaseOutputDataSource
# noqa
from
beat.backend.python.data
import
RemoteDataSource
# noqa
from
beat.backend.python.data
import
DataSink
# noqa
from
beat.backend.python.data
import
CachedDataSink
# noqa
from
beat.backend.python.data
import
StdoutDataSink
# noqa
from
beat.backend.python.data
import
load_data_index
# noqa
from
beat.backend.python.data
import
load_data_index_db
# noqa
from
beat.backend.python.data
import
foundSplitRanges
# noqa
beat/core/data_loaders.py
View file @
a1e373d4
...
...
@@ -46,6 +46,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
"""
from
beat.backend.python.data_loaders
import
DataLoaderList
from
beat.backend.python.data_loaders
import
DataLoader
from
beat.backend.python.data_loaders
import
DataView
from
beat.backend.python.data_loaders
import
DataLoaderList
# noqa
from
beat.backend.python.data_loaders
import
DataLoader
# noqa
from
beat.backend.python.data_loaders
import
DataView
# noqa
beat/core/dataformat.py
View file @
a1e373d4
...
...
@@ -45,18 +45,14 @@ Forward importing from :py:mod:`beat.backend.python.dataformat`:
:py:class:`beat.backend.python.dataformat.Storage`
"""
import
os
import
copy
import
six
import
simplejson
from
.
import
schema
from
.
import
prototypes
from
.
import
utils
from
.baseformat
import
baseformat
from
beat.backend.python.dataformat
import
Storage
from
beat.backend.python.dataformat
import
Storage
# noqa
from
beat.backend.python.dataformat
import
DataFormat
as
BackendDataFormat
...
...
@@ -132,7 +128,7 @@ class DataFormat(BackendDataFormat):
if
data
is
None
:
# loads prototype and validates it
self
.
data
,
self
.
errors
=
prototypes
.
load
(
"dataformat"
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
# nosec
else
:
...
...
beat/core/dock.py
View file @
a1e373d4
...
...
@@ -49,7 +49,7 @@ import socket
import
tempfile
import
time
import
docker
import
subprocess
as
sp
import
subprocess
as
sp
# nosec
import
logging
from
beat.core
import
stats
...
...
@@ -140,14 +140,14 @@ class Host(object):
self
.
processing_environments
.
keys
(),
)
)[
0
]
except
:
except
IndexError
:
try
:
return
list
(
filter
(
lambda
x
:
x
.
startswith
(
name
+
" ("
),
self
.
db_environments
.
keys
()
)
)[
0
]
except
:
except
IndexError
:
return
None
@
property
...
...
@@ -245,10 +245,10 @@ class Host(object):
replacement
=
True
else
:
keep
=
True
except
:
except
Exception
:
replacement
=
True
except
:
except
Exception
:
keep
=
True
elif
new_version
is
not
None
:
...
...
@@ -493,7 +493,7 @@ class Host(object):
try
:
return
simplejson
.
loads
(
stdout
)[
0
][
"State"
][
"Status"
]
except
:
except
Exception
:
return
None
def
logs
(
self
,
container
):
...
...
@@ -569,7 +569,7 @@ class Host(object):
status
=
self
.
wait
(
container
)
output
=
self
.
logs
(
container
)
except
Exception
as
e
:
except
Exception
:
return
1
,
None
finally
:
...
...
beat/core/environments.py
View file @
a1e373d4
...
...
@@ -79,7 +79,7 @@ def enumerate_packages(host, environment_name):
package_lines
=
output
.
split
(
"
\n
"
)
for
package_line
in
package_lines
:
information
=
re
.
split
(
"\s+"
,
package_line
)
information
=
re
.
split
(
r
"\s+"
,
package_line
)
if
len
(
information
)
==
4
:
packages
.
append
({
"name"
:
information
[
0
],
"version"
:
information
[
1
]})
...
...
beat/core/execution/__init__.py
View file @
a1e373d4
...
...
@@ -33,7 +33,8 @@
# #
###################################################################################
# flake8: noqa
from
.docker
import
DockerExecutor
from
.local
import
LocalExecutor
from
.subprocess
import
SubprocessExecutor
from
.subprocess
import
SubprocessExecutor
# nosec
beat/core/execution/base.py
View file @
a1e373d4
...
...
@@ -44,14 +44,9 @@ Execution utilities
import
os
import
sys
import
glob
import
collections
import
logging
logger
=
logging
.
getLogger
(
__name__
)
import
simplejson
from
..
import
schema
...
...
@@ -62,6 +57,9 @@ from .. import stats
from
beat.backend.python.helpers
import
convert_experiment_configuration_to_container
logger
=
logging
.
getLogger
(
__name__
)
class
BaseExecutor
(
object
):
"""Executors runs the code given an execution block information
...
...
beat/core/execution/docker.py
View file @
a1e373d4
...
...
@@ -47,8 +47,6 @@ import shutil
import
logging
import
requests
import
simplejson
import
zmq
import
docker
from
beat.backend.python.execution
import
MessageHandler
...
...
@@ -203,7 +201,7 @@ class DockerExecutor(RemoteExecutor):
# Determine the docker image to use for the databases
try
:
databases_environment
=
self
.
host
.
db2docker
(
database_paths
.
keys
())
except
:
except
Exception
:
raise
RuntimeError
(
"No environment found for the databases `%s' "
"- available environments are %s"
...
...
beat/core/execution/local.py
View file @
a1e373d4
...
...
@@ -49,24 +49,17 @@ import tempfile
import
shutil
import
zmq
import
time
import
logging
logger
=
logging
.
getLogger
(
__name__
)
import
simplejson
from
.base
import
BaseExecutor
from
beat.backend.python.helpers
import
create_inputs_from_configuration
from
beat.backend.python.helpers
import
create_outputs_from_configuration
from
beat.backend.python.helpers
import
AccessMode
from
beat.backend.python.execution
import
AlgorithmExecutor
from
beat.backend.python.execution
import
MessageHandler
from
beat.backend.python.execution
import
LoopExecutor
from
beat.backend.python.execution
import
LoopMessageHandler
from
.base
import
BaseExecutor
logger
=
logging
.
getLogger
(
__name__
)
class
LocalExecutor
(
BaseExecutor
):
"""LocalExecutor runs the code given an execution block information
...
...
beat/core/execution/remote.py
View file @
a1e373d4
...
...
@@ -42,17 +42,16 @@ remote
Execution utilities
"""
import
zmq
import
logging
logger
=
logging
.
getLogger
(
__name__
)
from
.base
import
BaseExecutor
from
beat.backend.python.helpers
import
create_inputs_from_configuration
from
beat.backend.python.helpers
import
create_outputs_from_configuration
from
beat.backend.python.helpers
import
AccessMode
logger
=
logging
.
getLogger
(
__name__
)
class
RemoteExecutor
(
BaseExecutor
):
...
...
beat/core/execution/subprocess.py
View file @
a1e373d4
...
...
@@ -41,16 +41,12 @@ subprocess
Execution utilities
"""
from
__future__
import
absolute_import
import
os
import
shutil
import
logging
import
sys
import
subprocess
as
sp
import
subprocess
as
sp
# nosec
import
tempfile
import
requests
import
zmq
from
beat.backend.python.execution
import
MessageHandler
...
...
@@ -415,7 +411,7 @@ class SubprocessExecutor(RemoteExecutor):
algorithm_process
.
kill
()
status
=
algorithm_process
.
wait
()
except
:
except
Exception
:
algorithm_process
.
kill
()
status
=
algorithm_process
.
wait
()
...
...
beat/core/experiment.py
View file @
a1e373d4
...
...
@@ -231,9 +231,9 @@ class Experiment(object):
if
data
is
None
:
# loads prototype and validates it
experiment_data
,
self
.
errors
=
prototypes
.
load
(
"experiment"
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
# nosec
toolchain_data
,
self
.
errors
=
prototypes
.
load
(
"toolchain"
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
# nosec
elif
isinstance
(
data
,
(
tuple
,
list
)):
# the user has passed a tuple
...
...
@@ -477,7 +477,7 @@ class Experiment(object):
self
.
errors
.
append
(
"/loop/%s/nb_slots: you have set the number "
"of slots for algorithm `%s' to %d, but it is not "
"splittable"
%
(
a
nalyzer
name
,
thisalgo
.
name
,
loop
[
"nb_slots"
])
"splittable"
%
(
a
lgo
name
,
thisalgo
.
name
,
loop
[
"nb_slots"
])
)
# check parameter consistence
...
...
@@ -740,7 +740,7 @@ class Experiment(object):
from_dtype
=
self
.
algorithms
[
block
[
"algorithm"
]].
output_map
[
algout
]
from_name
=
"block"
else
:
self
.
errors
.
append
(
"Unknown endpoint %s"
%
to
_endpt
[
0
])
self
.
errors
.
append
(
"Unknown endpoint %s"
%
from
_endpt
[
0
])
continue
to_endpt
=
connection
[
"to"
].
split
(
"."
,
1
)
...
...
@@ -869,7 +869,7 @@ class Experiment(object):
# for the grouping properties for the inputs
# create channel groups
chain_
in
=
collections
.
Counter
(
input_connections
)
chain_
groups
=
collections
.
Counter
(
input_connections
)
# now check the algorithm for conformance
algo_groups
=
self
.
algorithms
[
self
.
analyzers
[
name
][
"algorithm"
]].
groups
...
...
@@ -926,7 +926,7 @@ class Experiment(object):
)
# makes sure we don't have multiple incomming connections
assert
len
(
_connections
)
==
len
(
connections
),
(
assert
len
(
_connections
)
==
len
(
connections
),
(
# nosec
"detected multiple input "
"connections for block `%s' on experiment `%s'"
%
(
name
,
self
.
label
)
)
...
...
@@ -975,7 +975,6 @@ class Experiment(object):
# then go one by one generating the input **and** output hashes
# until all is done.
block_config
=
self
.
blocks
[
block
]
retval
[
algo_endpt
]
=
{
"from"
:
"%s.%s"
%
(
block
,
output
),
"channel"
:
channel
,
...
...
@@ -1181,8 +1180,6 @@ class Experiment(object):
def
dot_diagram
(
self
):
"""Returns a dot diagram representation of the experiment"""
from
.drawing
import
create_port_table
title
=
"Experiment: %s"
%
self
.
label
def
__label_callback
(
type
,
name
):
...
...
beat/core/hash.py
View file @
a1e373d4
...
...
@@ -48,8 +48,8 @@ import collections
import
simplejson
from
beat.backend.python.hash
import
*
from
beat.backend.python.hash
import
_sha256
from
beat.backend.python.hash
import
*
# noqa
from
beat.backend.python.hash
import
_sha256
# noqa
from
beat.backend.python.hash
import
_stringify
from
beat.backend.python.hash
import
_compact
...
...
@@ -164,7 +164,7 @@ def hashJSONStr(contents, description):
"""
try
:
return
hashJSON
(
return
hashJSON
(
# noqa
simplejson
.
loads
(
contents
,
object_pairs_hook
=
collections
.
OrderedDict
),
description
,
)
# preserve order
...
...
beat/core/inputs.py
View file @
a1e373d4
...
...
@@ -47,6 +47,6 @@ Forward imported from :py:mod:`beat.backend.python.inputs`:
"""
from
beat.backend.python.inputs
import
InputList
from
beat.backend.python.inputs
import
Input
from
beat.backend.python.inputs
import
InputGroup
from
beat.backend.python.inputs
import
InputList
# noqa
from
beat.backend.python.inputs
import
Input
# noqa
from
beat.backend.python.inputs
import
InputGroup
# noqa
beat/core/library.py
View file @
a1e373d4
...
...
@@ -147,7 +147,7 @@ class Library(BackendLibrary):
# At this point, `data' can be a dictionary or ``None``
if
data
is
None
:
# loads the default declaration for an library
self
.
data
,
self
.
errors
=
prototypes
.
load
(
"library"
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
# nosec
else
:
# just assign it
# this runs basic validation, including JSON loading if required
self
.
data
,
self
.
errors
=
schema
.
validate
(
"library"
,
data
)
...
...
beat/core/loader.py
View file @
a1e373d4
...
...
@@ -43,4 +43,4 @@ Forward imports from :py:mod:`beat.backend.python.loader`
"""
from
beat.backend.python.loader
import
*
from
beat.backend.python.loader
import
*
# noqa
beat/core/outputs.py
View file @
a1e373d4
...
...
@@ -46,6 +46,6 @@ Forward imported from :py:mod:`beat.backend.python.outputs`:
"""
from
beat.backend.python.outputs
import
SynchronizationListener
from
beat.backend.python.outputs
import
Output
from
beat.backend.python.outputs
import
OutputList
from
beat.backend.python.outputs
import
SynchronizationListener
# noqa
from
beat.backend.python.outputs
import
Output
# noqa
from
beat.backend.python.outputs
import
OutputList
# noqa
Prev
1
2
3
4
5
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment