Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
beat.backend.python
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
5
Issues
5
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
beat
beat.backend.python
Commits
b8fd9960
Commit
b8fd9960
authored
Jul 01, 2020
by
Samuel GAIST
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[backend] Pre-commit cleanup
parent
d1be9948
Pipeline
#40902
passed with stage
in 5 minutes and 2 seconds
Changes
14
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
225 additions
and
186 deletions
+225
-186
beat/backend/python/algorithm.py
beat/backend/python/algorithm.py
+2
-3
beat/backend/python/baseformat.py
beat/backend/python/baseformat.py
+150
-110
beat/backend/python/data.py
beat/backend/python/data.py
+12
-11
beat/backend/python/data_loaders.py
beat/backend/python/data_loaders.py
+0
-1
beat/backend/python/database.py
beat/backend/python/database.py
+7
-8
beat/backend/python/dataformat.py
beat/backend/python/dataformat.py
+2
-3
beat/backend/python/hash.py
beat/backend/python/hash.py
+26
-18
beat/backend/python/helpers.py
beat/backend/python/helpers.py
+7
-7
beat/backend/python/inputs.py
beat/backend/python/inputs.py
+0
-1
beat/backend/python/library.py
beat/backend/python/library.py
+1
-1
beat/backend/python/loader.py
beat/backend/python/loader.py
+5
-5
beat/backend/python/protocoltemplate.py
beat/backend/python/protocoltemplate.py
+1
-3
beat/backend/python/stats.py
beat/backend/python/stats.py
+10
-13
beat/backend/python/utils.py
beat/backend/python/utils.py
+2
-2
No files found.
beat/backend/python/algorithm.py
View file @
b8fd9960
...
...
@@ -42,20 +42,19 @@ algorithm
Validation for algorithms
"""
import
logging
import
os
import
sys
import
logging
import
six
import
numpy
import
simplejson
as
json
import
six
from
.
import
dataformat
from
.
import
library
from
.
import
loader
from
.
import
utils
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/backend/python/baseformat.py
View file @
b8fd9960
This diff is collapsed.
Click to expand it.
beat/backend/python/data.py
View file @
b8fd9960
...
...
@@ -42,23 +42,24 @@ data
Data I/O classes and functions
"""
import
abc
import
glob
import
logging
import
os
import
re
import
glob
import
simplejson
as
json
import
time
import
abc
import
zmq
import
logging
import
six
from
functools
import
reduce
from
collections
import
namedtuple
from
functools
import
reduce
import
simplejson
as
json
import
six
import
zmq
from
.hash
import
hashFileContents
from
.dataformat
import
DataFormat
from
.algorithm
import
Algorithm
from
.dataformat
import
DataFormat
from
.exceptions
import
RemoteException
from
.hash
import
hashFileContents
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -107,8 +108,8 @@ def mixDataIndices(list_of_data_indices):
for
index
in
range
(
start
,
end
+
1
):
done
=
False
for
l
in
list_of_data_indices
:
for
indices
in
l
:
for
data_indices
in
list_of_data_indices
:
for
indices
in
data_indices
:
if
indices
[
1
]
==
index
:
result
.
append
((
current_start
,
index
))
current_start
=
index
+
1
...
...
beat/backend/python/data_loaders.py
View file @
b8fd9960
...
...
@@ -46,7 +46,6 @@ import six
from
.data
import
mixDataIndices
# ----------------------------------------------------------
...
...
beat/backend/python/database.py
View file @
b8fd9960
...
...
@@ -42,23 +42,22 @@ database
Validation of databases
"""
import
itertools
import
os
import
sys
import
six
import
simplejson
as
json
import
itertools
import
numpy
as
np
from
collections
import
namedtuple
import
numpy
as
np
import
simplejson
as
json
import
six
from
.
import
loader
from
.
import
utils
from
.protocoltemplate
import
ProtocolTemplate
from
.dataformat
import
DataFormat
from
.outputs
import
OutputList
from
.exceptions
import
OutputError
from
.outputs
import
OutputList
from
.protocoltemplate
import
ProtocolTemplate
# ----------------------------------------------------------
...
...
beat/backend/python/dataformat.py
View file @
b8fd9960
...
...
@@ -42,17 +42,16 @@ dataformat
Validation and parsing for dataformats
"""
import
re
import
copy
import
re
import
six
import
numpy
import
simplejson
as
json
import
six
from
.
import
utils
from
.baseformat
import
baseformat
# ----------------------------------------------------------
...
...
beat/backend/python/hash.py
View file @
b8fd9960
...
...
@@ -42,13 +42,13 @@ hash
Various functions for hashing platform contributions and others
"""
import
hashlib
import
simplejson
import
collections
import
copy
import
six
import
hashlib
import
os
import
simplejson
import
six
# ----------------------------------------------------------
...
...
@@ -59,9 +59,9 @@ def _sha256(s):
"""
if
isinstance
(
s
,
six
.
string_types
):
try
:
s
=
six
.
u
(
s
).
encode
(
'utf-8'
)
s
=
six
.
u
(
s
).
encode
(
"utf-8"
)
except
Exception
:
s
=
s
.
encode
(
'utf-8'
)
s
=
s
.
encode
(
"utf-8"
)
return
hashlib
.
sha256
(
s
).
hexdigest
()
...
...
@@ -71,14 +71,14 @@ def _sha256(s):
def
_stringify
(
dictionary
):
names
=
sorted
(
dictionary
.
keys
())
converted_dictionary
=
'{'
converted_dictionary
=
"{"
for
name
in
names
:
converted_dictionary
+=
'"%s":%s,'
%
(
name
,
str
(
dictionary
[
name
]))
if
len
(
converted_dictionary
)
>
1
:
converted_dictionary
=
converted_dictionary
[:
-
1
]
converted_dictionary
+=
'}'
converted_dictionary
+=
"}"
return
converted_dictionary
...
...
@@ -87,13 +87,13 @@ def _stringify(dictionary):
def
_compact
(
text
):
return
text
.
replace
(
' '
,
''
).
replace
(
'
\
n
'
,
''
)
return
text
.
replace
(
" "
,
""
).
replace
(
"
\
n
"
,
""
)
# ----------------------------------------------------------
def
toPath
(
hash
,
suffix
=
'.data'
):
def
toPath
(
hash
,
suffix
=
".data"
):
""" Returns the path on disk which corresponds to the hash given.
Parameters:
...
...
@@ -159,7 +159,7 @@ def hashJSON(contents, description):
contents
=
copy
.
deepcopy
(
contents
)
# temporary copy
del
contents
[
description
]
contents
=
simplejson
.
dumps
(
contents
,
sort_keys
=
True
)
return
hashlib
.
sha256
(
contents
.
encode
(
'utf-8'
)).
hexdigest
()
return
hashlib
.
sha256
(
contents
.
encode
(
"utf-8"
)).
hexdigest
()
# ----------------------------------------------------------
...
...
@@ -175,11 +175,14 @@ def hashJSONFile(path, description):
"""
try
:
with
open
(
path
,
'rb'
)
as
f
:
with
open
(
path
,
"rb"
)
as
f
:
# preserve order
return
hashJSON
(
simplejson
.
loads
(
f
.
read
().
decode
(
'utf-8'
),
object_pairs_hook
=
collections
.
OrderedDict
),
description
)
return
hashJSON
(
simplejson
.
loads
(
f
.
read
().
decode
(
"utf-8"
),
object_pairs_hook
=
collections
.
OrderedDict
),
description
,
)
except
simplejson
.
JSONDecodeError
:
# falls back to normal file content hashing
return
hashFileContents
(
path
)
...
...
@@ -195,9 +198,9 @@ def hashFileContents(path):
str: hash
"""
with
open
(
path
,
'rb'
)
as
f
:
with
open
(
path
,
"rb"
)
as
f
:
sha256
=
hashlib
.
sha256
()
for
chunk
in
iter
(
lambda
:
f
.
read
(
sha256
.
block_size
*
1000
),
b
''
):
for
chunk
in
iter
(
lambda
:
f
.
read
(
sha256
.
block_size
*
1000
),
b
""
):
sha256
.
update
(
chunk
)
return
sha256
.
hexdigest
()
...
...
@@ -218,9 +221,14 @@ def hashDataset(database_name, protocol_name, set_name):
str: hash
"""
s
=
_compact
(
"""{
s
=
(
_compact
(
"""{
"database": "%s",
"protocol": "%s",
"set": "%s"
}"""
)
%
(
database_name
,
protocol_name
,
set_name
)
}"""
)
%
(
database_name
,
protocol_name
,
set_name
)
)
return
hash
(
s
)
beat/backend/python/helpers.py
View file @
b8fd9960
...
...
@@ -42,24 +42,24 @@ helpers
This module implements various helper methods and classes
"""
import
os
import
errno
import
logging
import
os
from
.algorithm
import
Algorithm
from
.data
import
CachedDataSink
from
.data
import
CachedDataSource
from
.data
import
RemoteDataSource
from
.data
import
CachedDataSink
from
.data
import
getAllFilenames
from
.data_loaders
import
DataLoaderList
from
.data_loaders
import
DataLoader
from
.
inputs
import
Input
List
from
.
data_loaders
import
DataLoader
List
from
.inputs
import
Input
from
.inputs
import
InputGroup
from
.outputs
import
SynchronizationListener
from
.outputs
import
OutputList
from
.inputs
import
InputList
from
.outputs
import
Output
from
.outputs
import
OutputList
from
.outputs
import
RemotelySyncedOutput
from
.
algorithm
import
Algorithm
from
.
outputs
import
SynchronizationListener
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/backend/python/inputs.py
View file @
b8fd9960
...
...
@@ -46,7 +46,6 @@ from functools import reduce
import
six
# ----------------------------------------------------------
...
...
beat/backend/python/library.py
View file @
b8fd9960
...
...
@@ -43,12 +43,12 @@ Validation for libraries
"""
import
os
import
simplejson
as
json
from
.
import
loader
from
.
import
utils
# ----------------------------------------------------------
...
...
beat/backend/python/loader.py
View file @
b8fd9960
...
...
@@ -44,10 +44,10 @@ executor. Safe in this context means that if the method raises an
exception, it will catch it and return in a suitable form to the caller.
"""
import
sys
import
six
import
imp
import
sys
import
six
# ----------------------------------------------------------
...
...
@@ -79,11 +79,11 @@ def load_module(name, path, uses):
# loads used modules
for
k
,
v
in
uses
.
items
():
retval
.
__dict__
[
k
]
=
load_module
(
k
,
v
[
'path'
],
v
[
'uses'
])
retval
.
__dict__
[
k
]
=
load_module
(
k
,
v
[
"path"
],
v
[
"uses"
])
# execute the module code on the context of previously import modules
with
open
(
path
,
"rb"
)
as
f
:
exec
(
compile
(
f
.
read
(),
path
,
'exec'
),
retval
.
__dict__
)
exec
(
compile
(
f
.
read
(),
path
,
"exec"
),
retval
.
__dict__
)
# nosec
return
retval
...
...
@@ -118,7 +118,7 @@ def run(obj, method, exc=None, *args, **kwargs):
"""
try
:
if
method
==
'__new__'
:
if
method
==
"__new__"
:
return
obj
(
*
args
,
**
kwargs
)
return
getattr
(
obj
,
method
)(
*
args
,
**
kwargs
)
...
...
beat/backend/python/protocoltemplate.py
View file @
b8fd9960
...
...
@@ -44,10 +44,8 @@ Validation of database protocol templates
import
simplejson
as
json
from
.dataformat
import
DataFormat
from
.
import
utils
from
.dataformat
import
DataFormat
# ----------------------------------------------------------
...
...
beat/backend/python/stats.py
View file @
b8fd9960
...
...
@@ -89,17 +89,14 @@ def io_statistics(configuration, input_list=None, output_list=None):
write_time
+=
duration
blocks_written
+=
output
.
nb_data_blocks_written
if
'result'
in
configuration
:
hash
=
configuration
[
'result'
][
'path'
].
replace
(
'/'
,
''
)
if
"result"
in
configuration
:
hash
=
configuration
[
"result"
][
"path"
].
replace
(
"/"
,
""
)
else
:
hash
=
configuration
[
'outputs'
][
output
.
name
][
'path'
].
replace
(
'/'
,
''
)
hash
=
configuration
[
"outputs"
][
output
.
name
][
"path"
].
replace
(
"/"
,
""
)
files
.
append
(
dict
(
hash
=
hash
,
size
=
size
,
blocks
=
output
.
nb_data_blocks_written
,
))
files
.
append
(
dict
(
hash
=
hash
,
size
=
size
,
blocks
=
output
.
nb_data_blocks_written
,)
)
# Result
return
dict
(
...
...
@@ -128,13 +125,13 @@ def update(statistics, additional_statistics):
"""
for
k
in
statistics
.
keys
():
if
k
==
'files'
:
if
k
==
"files"
:
continue
for
k2
in
statistics
[
k
].
keys
():
statistics
[
k
][
k2
]
+=
additional_statistics
[
k
][
k2
]
if
'files'
in
statistics
:
statistics
[
'files'
].
extend
(
additional_statistics
.
get
(
'files'
,
[]))
if
"files"
in
statistics
:
statistics
[
"files"
].
extend
(
additional_statistics
.
get
(
"files"
,
[]))
else
:
statistics
[
'files'
]
=
additional_statistics
.
get
(
'files'
,
[])
statistics
[
"files"
]
=
additional_statistics
.
get
(
"files"
,
[])
beat/backend/python/utils.py
View file @
b8fd9960
...
...
@@ -42,16 +42,16 @@ utils
This module implements helper classes and functions.
"""
import
collections
import
os
import
shutil
import
collections
import
numpy
import
simplejson
import
six
from
.
import
hash
# ----------------------------------------------------------
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment