Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
057a240d
Commit
057a240d
authored
Aug 06, 2020
by
Amir MOHAMMADI
Browse files
Merge branch 'pre_commit_cleanup' into 'master'
Pre commit cleanup See merge request
!126
parents
8ea89087
4c771e17
Pipeline
#41802
passed with stages
in 39 minutes and 6 seconds
Changes
85
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
.flake8
View file @
057a240d
[flake8]
max-line-length = 8
0
max-line-length = 8
8
select = B,C,E,F,W,T4,B9,B950
ignore = E501, W503, E203
.isort.cfg
0 → 100644
View file @
057a240d
[settings]
multi_line_output=3
include_trailing_comma=true
force_grid_wrap=0
use_parentheses=true
ensure_newline_before_comments=true
line_length=88
force_single_line=true
.pre-commit-config.yaml
View file @
057a240d
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos
:
-
repo
:
https://github.com/
ambv/black
rev
:
stable
-
repo
:
https://github.com/
timothycrosley/isort
rev
:
5.0.4
hooks
:
-
id
:
black
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
id
:
isort
files
:
.*.py
-
repo
:
https://github.com/psf/black
rev
:
19.10b0
hooks
:
-
id
:
black
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
repo
:
https://github.com/pre-commit/pre-commit-hooks
rev
:
v
2.0
.0
rev
:
v
3.1
.0
hooks
:
-
id
:
check-ast
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
id
:
check-case-conflict
-
id
:
trailing-whitespace
-
id
:
end-of-file-fixer
-
id
:
debug-statements
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
-
id
:
check-added-large-files
-
id
:
check-docstring-first
-
id
:
flake8
-
id
:
check-yaml
exclude
:
conda/meta.yaml
-
repo
:
https://github.com/PyCQA/flake8
rev
:
3.8.3
hooks
:
-
id
:
flake8
-
repo
:
https://github.com/PyCQA/bandit
rev
:
'
master'
# Update me!
rev
:
1.6.2
hooks
:
-
id
:
bandit
exclude
:
beat/core/test/prefix/algorithms/errors/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
...
...
beat/__init__.py
View file @
057a240d
...
...
@@ -33,8 +33,8 @@
# #
###################################################################################
# see https://docs.python.org/3/library/pkgutil.html
from
pkgutil
import
extend_path
__path__
=
extend_path
(
__path__
,
__name__
)
beat/core/algorithm.py
View file @
057a240d
...
...
@@ -45,22 +45,21 @@ Forward importing from :py:mod:`beat.backend.python.algorithm`
:py:class:`beat.backend.python.algorithm.Storage`
:py:class:`beat.backend.python.algorithm.Runner`
"""
import
os
import
six
import
numpy
import
pkg_resources
import
simplejson
as
json
import
six
from
beat.backend.python.algorithm
import
Algorithm
as
BackendAlgorithm
from
beat.backend.python.algorithm
import
Runner
# noqa
from
beat.backend.python.algorithm
import
Storage
from
.
import
dataformat
from
.
import
library
from
.
import
schema
from
.
import
prototypes
from
beat.backend.python.algorithm
import
Storage
from
beat.backend.python.algorithm
import
Runner
# noqa
from
beat.backend.python.algorithm
import
Algorithm
as
BackendAlgorithm
from
.
import
schema
def
load_algorithm_prototype
(
prefix
):
...
...
beat/core/bcp/broker.py
View file @
057a240d
...
...
@@ -49,15 +49,14 @@ Options:
-V, --version Show version
-v, --verbose Increases the output verbosity level
"""
import
os
import
sys
from
docopt
import
docopt
from
..bcpapi.broker
import
BeatComputationBroker
from
..version
import
__version__
from
..utils
import
setup_logging
from
..version
import
__version__
def
run
(
port
=
5555
,
verbose
=
1
,
callbacks
=
None
):
...
...
beat/core/bcp/worker.py
View file @
057a240d
...
...
@@ -61,28 +61,25 @@ Options:
--cache-mount-point=<cache_mount_point> NFS mount point to use for cache setup
"""
import
os
import
sys
import
signal
import
sys
import
tempfile
import
simplejson
as
json
from
socket
import
gethostname
import
simplejson
as
json
import
zmq
from
socket
import
gethostname
from
docopt
import
docopt
from
..bcpapi
import
BCP
from
..bcpapi.worker
import
BeatComputationWorker
from
..bcpapi.processor
import
BeatComputationProcessor
from
..bcpapi.execution
import
ExecutionProcess
from
..bcpapi.processor
import
BeatComputationProcessor
from
..bcpapi.worker
import
BeatComputationWorker
from
..dock
import
Host
from
..utils
import
find_free_port
from
..utils
import
setup_logging
from
..version
import
__version__
logger
=
None
...
...
beat/core/bcpapi/broker.py
View file @
057a240d
...
...
@@ -37,20 +37,16 @@
Inspired by the Majordomo Protocol Broker
"""
import
logging
import
time
import
signal
import
time
from
binascii
import
hexlify
import
zmq
# local
from
.
import
BCP
from
.zhelpers
import
dump
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/core/bcpapi/execution.py
View file @
057a240d
...
...
@@ -40,17 +40,16 @@ management
Execution utilities
"""
import
logging
import
multiprocessing
import
signal
import
zmq
import
simplejson
as
json
import
zmq
from
..dock
import
Host
from
..execution.local
import
LocalExecutor
from
..execution.docker
import
DockerExecutor
from
..execution.local
import
LocalExecutor
from
.
import
BCP
...
...
beat/core/bcpapi/processor.py
View file @
057a240d
...
...
@@ -34,8 +34,8 @@
"""BEAT Computation worker"""
import
logging
import
zmq
from
.zhelpers
import
dump
...
...
beat/core/bcpapi/worker.py
View file @
057a240d
...
...
@@ -34,14 +34,12 @@
"""BEAT Computation worker"""
import
logging
import
time
import
zmq
# BEAT Computation protocol constants:
from
.
import
BCP
from
.zhelpers
import
dump
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -58,8 +56,8 @@ class BeatComputationWorker(object):
service
=
service
.
encode
(
"utf-8"
)
self
.
heartbeat_at
=
(
0
)
# When to send HEARTBEAT (relative to time.time(), so in seconds)
0
# When to send HEARTBEAT (relative to time.time(), so in seconds)
)
self
.
liveness
=
0
# How many attempts left
self
.
heartbeat
=
2500
# Heartbeat delay, msecs
self
.
reconnect
=
2500
# Reconnect delay, msecs
...
...
beat/core/bcpapi/zhelpers.py
View file @
057a240d
...
...
@@ -38,9 +38,9 @@ Helper module for common zmq task
Based on Majordomo protocol zhelpers.
"""
import
binascii
import
logging
import
zmq
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/core/data.py
View file @
057a240d
...
...
@@ -53,16 +53,15 @@ Forward importing from :py:mod:`beat.backend.python.data`:
:py:func:`beat.backend.python.data.load_data_index_db`
:py:func:`beat.backend.python.data.foundSplitRanges`
"""
from
beat.backend.python.data
import
mixDataIndices
# noqa
from
beat.backend.python.data
import
getAllFilenames
# noqa
from
beat.backend.python.data
import
DataSource
# noqa
from
beat.backend.python.data
import
CachedDataSink
# noqa
from
beat.backend.python.data
import
CachedDataSource
# noqa
from
beat.backend.python.data
import
DatabaseOutputDataSource
# noqa
from
beat.backend.python.data
import
RemoteDataSource
# noqa
from
beat.backend.python.data
import
DataSink
# noqa
from
beat.backend.python.data
import
CachedDataSink
# noqa
from
beat.backend.python.data
import
DataSource
# noqa
from
beat.backend.python.data
import
RemoteDataSource
# noqa
from
beat.backend.python.data
import
StdoutDataSink
# noqa
from
beat.backend.python.data
import
foundSplitRanges
# noqa
from
beat.backend.python.data
import
getAllFilenames
# noqa
from
beat.backend.python.data
import
load_data_index
# noqa
from
beat.backend.python.data
import
load_data_index_db
# noqa
from
beat.backend.python.data
import
foundSplitRang
es
# noqa
from
beat.backend.python.data
import
mixDataIndic
es
# noqa
beat/core/data_loaders.py
View file @
057a240d
...
...
@@ -44,8 +44,6 @@ Forward importing from :py:mod:`beat.backend.python.data_loaders`
:py:class:`beat.backend.python.data_loaders.DataLoader`
:py:class:`beat.backend.python.data_loaders.DataView`
"""
from
beat.backend.python.data_loaders
import
DataLoaderList
# noqa
from
beat.backend.python.data_loaders
import
DataLoader
# noqa
from
beat.backend.python.data_loaders
import
DataLoaderList
# noqa
from
beat.backend.python.data_loaders
import
DataView
# noqa
beat/core/database.py
View file @
057a240d
...
...
@@ -44,22 +44,19 @@ Validation of databases
Forward importing from :py:mod:`beat.backend.python.database`:
:py:class:`beat.backend.python.database.Storage`
"""
import
os
import
six
from
.
import
schema
from
beat.backend.python.database
import
Database
as
BackendDatabase
from
beat.backend.python.database
import
Storage
from
beat.backend.python.protocoltemplate
import
Storage
as
PTStorage
from
.
import
prototypes
from
.
import
schema
from
.dataformat
import
DataFormat
from
.protocoltemplate
import
ProtocolTemplate
from
.
import
prototypes
from
beat.backend.python.database
import
Storage
from
beat.backend.python.database
import
Database
as
BackendDatabase
from
beat.backend.python.protocoltemplate
import
Storage
as
PTStorage
def
get_first_procotol_template
(
prefix
):
pt_root_folder
=
os
.
path
.
join
(
prefix
,
PTStorage
.
asset_folder
)
...
...
beat/core/dataformat.py
View file @
057a240d
...
...
@@ -44,17 +44,17 @@ Validation and parsing for dataformats
Forward importing from :py:mod:`beat.backend.python.dataformat`:
:py:class:`beat.backend.python.dataformat.Storage`
"""
import
copy
import
six
from
.
import
schema
from
beat.backend.python.dataformat
import
DataFormat
as
BackendDataFormat
from
beat.backend.python.dataformat
import
Storage
# noqa
from
.
import
prototypes
from
.
import
schema
from
.
import
utils
from
beat.backend.python.dataformat
import
Storage
# noqa
from
beat.backend.python.dataformat
import
DataFormat
as
BackendDataFormat
class
DataFormat
(
BackendDataFormat
):
"""Data formats define the chunks of data that circulate between blocks.
...
...
beat/core/dock.py
View file @
057a240d
...
...
@@ -41,18 +41,16 @@ dock
Docker helper classes
"""
import
ast
import
logging
import
os
import
simplejson
as
json
import
socket
import
subprocess
as
sp
# nosec
import
tempfile
import
time
import
docker
import
subprocess
as
sp
# nosec
import
logging
import
docker
import
simplejson
as
json
from
packaging
import
version
from
beat.core
import
stats
...
...
beat/core/environments.py
View file @
057a240d
...
...
@@ -41,10 +41,8 @@ environment
Helper functions related to environment management
"""
import
re
import
logging
import
re
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/core/execution/base.py
View file @
057a240d
...
...
@@ -41,21 +41,19 @@ base
Execution utilities
"""
import
os
import
glob
import
collections
import
glob
import
logging
import
s
implejson
as
json
import
o
s
from
..
import
schema
from
..
import
database
from
..
import
algorithm
from
..
import
stats
import
simplejson
as
json
from
beat.backend.python.helpers
import
convert_experiment_configuration_to_container
from
..
import
algorithm
from
..
import
database
from
..
import
schema
from
..
import
stats
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/core/execution/docker.py
View file @
057a240d
...
...
@@ -41,24 +41,21 @@ docker
Execution utilities
"""
import
logging
import
os
import
shutil
import
logging
from
collections
import
namedtuple
import
requests
import
simplejson
as
json
from
collections
import
namedtuple
from
beat.backend.python.execution
import
MessageHandler
from
beat.backend.python.data
import
getAllFilenames
from
beat.backend.python.execution
import
MessageHandler
from
..
import
stats
from
..
import
utils
from
.remote
import
RemoteExecutor
logger
=
logging
.
getLogger
(
__name__
)
...
...
Prev
1
2
3
4
5
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment