Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
beat.backend.python
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
5
Issues
5
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
beat
beat.backend.python
Commits
39e23b21
Commit
39e23b21
authored
Jul 01, 2020
by
Samuel GAIST
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[test] Pre-commit cleanup
parent
a2c7f31b
Changes
25
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
25 changed files
with
446 additions
and
407 deletions
+446
-407
beat/backend/python/test/__init__.py
beat/backend/python/test/__init__.py
+2
-1
beat/backend/python/test/mocks.py
beat/backend/python/test/mocks.py
+16
-14
beat/backend/python/test/test_algorithm.py
beat/backend/python/test/test_algorithm.py
+8
-11
beat/backend/python/test/test_data.py
beat/backend/python/test/test_data.py
+7
-9
beat/backend/python/test/test_data_loaders.py
beat/backend/python/test/test_data_loaders.py
+5
-6
beat/backend/python/test/test_database.py
beat/backend/python/test/test_database.py
+1
-2
beat/backend/python/test/test_database_view.py
beat/backend/python/test/test_database_view.py
+4
-7
beat/backend/python/test/test_databases_index.py
beat/backend/python/test/test_databases_index.py
+5
-9
beat/backend/python/test/test_databases_provider.py
beat/backend/python/test/test_databases_provider.py
+10
-13
beat/backend/python/test/test_dbexecutor.py
beat/backend/python/test/test_dbexecutor.py
+7
-9
beat/backend/python/test/test_executor.py
beat/backend/python/test/test_executor.py
+49
-53
beat/backend/python/test/test_format_array.py
beat/backend/python/test/test_format_array.py
+1
-3
beat/backend/python/test/test_format_extends.py
beat/backend/python/test/test_format_extends.py
+1
-3
beat/backend/python/test/test_format_missing_attributes.py
beat/backend/python/test/test_format_missing_attributes.py
+0
-2
beat/backend/python/test/test_format_object.py
beat/backend/python/test/test_format_object.py
+1
-3
beat/backend/python/test/test_format_simple.py
beat/backend/python/test/test_format_simple.py
+1
-3
beat/backend/python/test/test_format_unsafe_cast.py
beat/backend/python/test/test_format_unsafe_cast.py
+1
-3
beat/backend/python/test/test_format_valid_cast.py
beat/backend/python/test/test_format_valid_cast.py
+1
-3
beat/backend/python/test/test_hash.py
beat/backend/python/test/test_hash.py
+0
-1
beat/backend/python/test/test_helpers.py
beat/backend/python/test/test_helpers.py
+6
-7
beat/backend/python/test/test_inputs.py
beat/backend/python/test/test_inputs.py
+292
-212
beat/backend/python/test/test_loop_executor.py
beat/backend/python/test/test_loop_executor.py
+11
-14
beat/backend/python/test/test_message_handler.py
beat/backend/python/test/test_message_handler.py
+9
-10
beat/backend/python/test/test_outputs.py
beat/backend/python/test/test_outputs.py
+7
-7
beat/backend/python/test/test_protocoltemplate.py
beat/backend/python/test/test_protocoltemplate.py
+1
-2
No files found.
beat/backend/python/test/__init__.py
View file @
39e23b21
...
...
@@ -34,4 +34,5 @@
###################################################################################
import
pkg_resources
prefix
=
pkg_resources
.
resource_filename
(
__name__
,
'prefix'
)
prefix
=
pkg_resources
.
resource_filename
(
__name__
,
"prefix"
)
beat/backend/python/test/mocks.py
View file @
39e23b21
...
...
@@ -38,42 +38,39 @@ import logging
from
collections
import
namedtuple
from
..data
import
DataSource
from
..data
import
DataSink
from
..data
import
DataSource
class
MockDataSource
(
DataSource
):
def
__init__
(
self
,
data_and_indices
):
super
(
MockDataSource
,
self
).
__init__
()
self
.
infos
=
[
(
x
[
1
],
x
[
2
])
for
x
in
data_and_indices
]
self
.
data
=
[
x
[
0
]
for
x
in
data_and_indices
]
self
.
infos
=
[
(
x
[
1
],
x
[
2
])
for
x
in
data_and_indices
]
self
.
data
=
[
x
[
0
]
for
x
in
data_and_indices
]
def
__getitem__
(
self
,
index
):
return
(
self
.
data
[
index
],
self
.
infos
[
index
][
0
],
self
.
infos
[
index
][
1
])
#----------------------------------------------------------
#
----------------------------------------------------------
class
CrashingDataSource
(
DataSource
):
def
__init__
(
self
):
super
(
CrashingDataSource
,
self
).
__init__
()
Infos
=
namedtuple
(
'Infos'
,
[
'start_index'
,
'end_index'
])
Infos
=
namedtuple
(
"Infos"
,
[
"start_index"
,
"end_index"
])
self
.
infos
=
[
Infos
(
0
,
0
)]
def
__getitem__
(
self
,
index
):
a
=
b
a
=
b
# noqa
return
(
None
,
None
,
None
)
#----------------------------------------------------------
#
----------------------------------------------------------
class
MockDataSink
(
DataSink
):
class
WrittenData
:
def
__init__
(
self
,
data
,
start
,
end
):
self
.
data
=
data
...
...
@@ -86,7 +83,8 @@ class MockDataSink(DataSink):
self
.
dataformat
=
dataformat
def
write
(
self
,
data
,
start_data_index
,
end_data_index
):
if
not
(
self
.
can_write
):
raise
IOError
if
not
(
self
.
can_write
):
raise
IOError
self
.
written
.
append
(
MockDataSink
.
WrittenData
(
data
,
start_data_index
,
end_data_index
)
)
...
...
@@ -94,6 +92,7 @@ class MockDataSink(DataSink):
def
isConnected
(
self
):
return
True
# Based on https://stackoverflow.com/a/20553331/5843716
class
MockLoggingHandler
(
logging
.
Handler
):
"""Mock logging handler to check for expected logs.
...
...
@@ -104,9 +103,12 @@ class MockLoggingHandler(logging.Handler):
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
messages
=
{
'debug'
:
[],
'info'
:
[],
'warning'
:
[],
'error'
:
[],
'critical'
:
[],
'extra'
:
[]
"debug"
:
[],
"info"
:
[],
"warning"
:
[],
"error"
:
[],
"critical"
:
[],
"extra"
:
[],
}
super
(
MockLoggingHandler
,
self
).
__init__
(
*
args
,
**
kwargs
)
...
...
beat/backend/python/test/test_algorithm.py
View file @
39e23b21
...
...
@@ -34,34 +34,31 @@
###################################################################################
import
unittest
import
six
import
os
import
glob
import
tempfile
import
logging
import
os
import
tempfile
import
unittest
import
numpy
as
np
import
six
from
..algorithm
import
Algorithm
from
..data_loaders
import
DataLoaderList
from
..data_loaders
import
DataLoader
from
..dataformat
import
DataFormat
from
..data
import
CachedDataSink
from
..data
import
CachedDataSource
from
..data_loaders
import
DataLoader
from
..data_loaders
import
DataLoaderList
from
..dataformat
import
DataFormat
from
..inputs
import
Input
from
..inputs
import
InputGroup
from
..inputs
import
InputList
from
..outputs
import
Output
from
..outputs
import
OutputList
from
..outputs
import
SynchronizationListener
from
.
import
prefix
from
.mocks
import
MockDataSink
from
.mocks
import
MockLoggingHandler
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_data.py
View file @
39e23b21
...
...
@@ -34,24 +34,22 @@
###################################################################################
import
unittest
import
os
import
glob
import
os
import
pickle
# nosec
import
tempfile
import
shutil
import
tempfile
import
unittest
from
..data
import
mixDataIndices
from
..data
import
CachedDataSource
from
..data
import
CachedDataSink
from
..data
import
getAllFilenames
from
..data
import
CachedDataSource
from
..data
import
foundSplitRanges
from
..dataformat
import
DataFormat
from
..data
import
getAllFilenames
from
..data
import
mixDataIndices
from
..database
import
Database
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_data_loaders.py
View file @
39e23b21
...
...
@@ -34,21 +34,20 @@
###################################################################################
import
unittest
import
os
import
glob
import
os
import
tempfile
import
unittest
import
numpy
as
np
from
..data
import
CachedDataSink
from
..data
import
CachedDataSource
from
..data_loaders
import
DataLoader
from
..data_loaders
import
DataLoaderList
from
..dataformat
import
DataFormat
from
..data
import
CachedDataSink
from
..data
import
CachedDataSource
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_database.py
View file @
39e23b21
...
...
@@ -35,11 +35,10 @@
import
nose.tools
from
..database
import
Database
from
..database
import
Database
from
.
import
prefix
INTEGERS_DBS
=
[
"integers_db/{}"
.
format
(
i
)
for
i
in
range
(
1
,
3
)]
...
...
beat/backend/python/test/test_database_view.py
View file @
39e23b21
...
...
@@ -34,20 +34,17 @@
###################################################################################
import
unittest
import
tempfile
import
shutil
import
os
import
shutil
import
tempfile
import
unittest
from
ddt
import
ddt
from
ddt
import
idata
from
..database
import
Database
from
.test_database
import
INTEGERS_DBS
from
.
import
prefix
from
.test_database
import
INTEGERS_DBS
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_databases_index.py
View file @
39e23b21
...
...
@@ -36,24 +36,20 @@
# Tests for experiment execution
import
os
import
unittest
import
multiprocessing
import
tempfile
import
os
import
shutil
import
tempfile
import
unittest
from
ddt
import
ddt
from
ddt
import
idata
from
..scripts
import
index
from
..hash
import
hashDataset
from
..hash
import
toPath
from
.test_database
import
INTEGERS_DBS
from
..scripts
import
index
from
.
import
prefix
from
.test_database
import
INTEGERS_DBS
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_databases_provider.py
View file @
39e23b21
...
...
@@ -36,30 +36,27 @@
# Tests for experiment execution
import
os
import
socket
import
unittest
import
json
import
multiprocessing
import
tempfile
import
os
import
shutil
import
socket
import
tempfile
import
unittest
from
contextlib
import
closing
import
zmq
from
ddt
import
ddt
from
ddt
import
idata
from
contextlib
import
closing
from
..scripts
import
databases_provider
from
..database
import
Database
from
..data
import
RemoteDataSource
from
..data
import
RemoteException
from
.test_database
import
INTEGERS_DBS
from
..database
import
Database
from
..scripts
import
databases_provider
from
.
import
prefix
from
.test_database
import
INTEGERS_DBS
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_dbexecutor.py
View file @
39e23b21
...
...
@@ -37,26 +37,24 @@
# Tests for experiment execution
import
os
import
shutil
import
tempfile
import
unittest
import
zmq
import
tempfile
import
shutil
from
ddt
import
ddt
from
ddt
import
idata
from
..data
import
RemoteDataSource
from
..data_loaders
import
DataLoader
from
..database
import
Database
from
..execution
import
DBExecutor
from
..execution
import
MessageHandler
from
..database
import
Database
from
..data_loaders
import
DataLoader
from
..data
import
RemoteDataSource
from
..hash
import
hashDataset
from
..hash
import
toPath
from
.test_database
import
INTEGERS_DBS
from
.
import
prefix
from
.test_database
import
INTEGERS_DBS
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_executor.py
View file @
39e23b21
...
...
@@ -34,55 +34,41 @@
###################################################################################
import
unittest
import
logging
import
tempfile
import
simplejson
import
os
import
zmq
import
shutil
import
numpy
as
np
import
tempfile
import
unittest
from
copy
import
deepcopy
from
..execution
import
AlgorithmExecutor
from
..execution
import
MessageHandler
from
..inputs
import
InputList
import
numpy
as
np
import
simplejson
import
zmq
from
..algorithm
import
Algorithm
from
..dataformat
import
DataFormat
from
..data
import
CachedDataSink
from
..data
import
CachedDataSource
from
..dataformat
import
DataFormat
from
..execution
import
AlgorithmExecutor
from
..execution
import
MessageHandler
from
..helpers
import
convert_experiment_configuration_to_container
from
..helpers
import
AccessMode
from
.
import
prefix
from
.mocks
import
MockLoggingHandler
CONFIGURATION
=
{
'algorithm'
:
''
,
'channel'
:
'main'
,
'parameters'
:
{
},
'inputs'
:
{
'in'
:
{
'path'
:
'INPUT'
,
'channel'
:
'main'
,
}
},
'outputs'
:
{
'out'
:
{
'path'
:
'OUTPUT'
,
'channel'
:
'main'
}
},
"algorithm"
:
""
,
"channel"
:
"main"
,
"parameters"
:
{},
"inputs"
:
{
"in"
:
{
"path"
:
"INPUT"
,
"channel"
:
"main"
}},
"outputs"
:
{
"out"
:
{
"path"
:
"OUTPUT"
,
"channel"
:
"main"
}},
}
#----------------------------------------------------------
#
----------------------------------------------------------
class
TestAlgorithmExecutor
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
cache_root
=
tempfile
.
mkdtemp
(
prefix
=
__name__
)
self
.
working_dir
=
tempfile
.
mkdtemp
(
prefix
=
__name__
)
...
...
@@ -90,7 +76,6 @@ class TestAlgorithmExecutor(unittest.TestCase):
self
.
executor_socket
=
None
self
.
zmq_context
=
None
def
tearDown
(
self
):
shutil
.
rmtree
(
self
.
cache_root
)
shutil
.
rmtree
(
self
.
working_dir
)
...
...
@@ -108,15 +93,18 @@ class TestAlgorithmExecutor(unittest.TestCase):
self
.
executor_socket
=
None
self
.
zmq_context
=
None
def
writeData
(
self
,
input_name
,
indices
,
start_value
):
filename
=
os
.
path
.
join
(
self
.
cache_root
,
CONFIGURATION
[
'inputs'
][
input_name
][
'path'
]
+
'.data'
)
filename
=
os
.
path
.
join
(
self
.
cache_root
,
CONFIGURATION
[
"inputs"
][
input_name
][
"path"
]
+
".data"
)
dataformat
=
DataFormat
(
prefix
,
'user/single_integer/1'
)
dataformat
=
DataFormat
(
prefix
,
"user/single_integer/1"
)
self
.
assertTrue
(
dataformat
.
valid
)
data_sink
=
CachedDataSink
()
self
.
assertTrue
(
data_sink
.
setup
(
filename
,
dataformat
,
indices
[
0
][
0
],
indices
[
-
1
][
1
]))
self
.
assertTrue
(
data_sink
.
setup
(
filename
,
dataformat
,
indices
[
0
][
0
],
indices
[
-
1
][
1
])
)
for
i
in
indices
:
data
=
dataformat
.
type
()
...
...
@@ -130,26 +118,25 @@ class TestAlgorithmExecutor(unittest.TestCase):
data_sink
.
close
()
del
data_sink
def
process
(
self
,
algorithm_name
):
self
.
writeData
(
'in'
,
[(
0
,
0
),
(
1
,
1
),
(
2
,
2
),
(
3
,
3
)],
1000
)
self
.
writeData
(
"in"
,
[(
0
,
0
),
(
1
,
1
),
(
2
,
2
),
(
3
,
3
)],
1000
)
config
=
deepcopy
(
CONFIGURATION
)
config
[
'algorithm'
]
=
algorithm_name
config
[
"algorithm"
]
=
algorithm_name
config
=
convert_experiment_configuration_to_container
(
config
)
with
open
(
os
.
path
.
join
(
self
.
working_dir
,
'configuration.json'
),
'wb'
)
as
f
:
data
=
simplejson
.
dumps
(
config
,
indent
=
4
).
encode
(
'utf-8'
)
with
open
(
os
.
path
.
join
(
self
.
working_dir
,
"configuration.json"
),
"wb"
)
as
f
:
data
=
simplejson
.
dumps
(
config
,
indent
=
4
).
encode
(
"utf-8"
)
f
.
write
(
data
)
working_prefix
=
os
.
path
.
join
(
self
.
working_dir
,
'prefix'
)
working_prefix
=
os
.
path
.
join
(
self
.
working_dir
,
"prefix"
)
if
not
os
.
path
.
exists
(
working_prefix
):
os
.
makedirs
(
working_prefix
)
algorithm
=
Algorithm
(
prefix
,
algorithm_name
)
algorithm
.
export
(
working_prefix
)
self
.
message_handler
=
MessageHandler
(
'127.0.0.1'
)
self
.
message_handler
=
MessageHandler
(
"127.0.0.1"
)
self
.
message_handler
.
start
()
...
...
@@ -157,14 +144,23 @@ class TestAlgorithmExecutor(unittest.TestCase):
self
.
executor_socket
=
self
.
zmq_context
.
socket
(
zmq
.
PAIR
)
self
.
executor_socket
.
connect
(
self
.
message_handler
.
address
)
executor
=
AlgorithmExecutor
(
self
.
executor_socket
,
self
.
working_dir
,
cache_root
=
self
.
cache_root
)
executor
=
AlgorithmExecutor
(
self
.
executor_socket
,
self
.
working_dir
,
cache_root
=
self
.
cache_root
)
self
.
assertTrue
(
executor
.
setup
())
self
.
assertTrue
(
executor
.
prepare
())
self
.
assertTrue
(
executor
.
process
())
cached_file
=
CachedDataSource
()
self
.
assertTrue
(
cached_file
.
setup
(
os
.
path
.
join
(
self
.
cache_root
,
CONFIGURATION
[
'outputs'
][
'out'
][
'path'
]
+
'.data'
),
prefix
))
self
.
assertTrue
(
cached_file
.
setup
(
os
.
path
.
join
(
self
.
cache_root
,
CONFIGURATION
[
"outputs"
][
"out"
][
"path"
]
+
".data"
),
prefix
,
)
)
for
i
in
range
(
len
(
cached_file
)):
data
,
start
,
end
=
cached_file
[
i
]
...
...
@@ -172,22 +168,22 @@ class TestAlgorithmExecutor(unittest.TestCase):
self
.
assertEqual
(
start
,
i
)
self
.
assertEqual
(
end
,
i
)
def
test_legacy_echo_1
(
self
):
log_handler
=
MockLoggingHandler
(
level
=
'DEBUG'
)
log_handler
=
MockLoggingHandler
(
level
=
"DEBUG"
)
logging
.
getLogger
().
addHandler
(
log_handler
)
log_messages
=
log_handler
.
messages
self
.
process
(
'legacy/echo/1'
)
self
.
process
(
"legacy/echo/1"
)
info_len
=
len
(
log_messages
[
'warning'
])
info_len
=
len
(
log_messages
[
"warning"
])
self
.
assertEqual
(
info_len
,
1
)
self
.
assertEqual
(
log_messages
[
'warning'
][
info_len
-
1
],
'legacy/echo/1 is using LEGACY I/O API, please upgrade this algorithm as soon as possible'
)
self
.
assertEqual
(
log_messages
[
"warning"
][
info_len
-
1
],
"legacy/echo/1 is using LEGACY I/O API, please upgrade this algorithm as soon as possible"
,
)
def
test_sequential_echo_1
(
self
):
self
.
process
(
'sequential/echo/1'
)
self
.
process
(
"sequential/echo/1"
)
def
test_autonomous_echo_1
(
self
):
self
.
process
(
'autonomous/echo/1'
)
self
.
process
(
"autonomous/echo/1"
)
beat/backend/python/test/test_format_array.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
numpy
import
nose.tools
import
numpy
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_format_extends.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
numpy
import
nose.tools
import
numpy
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_format_missing_attributes.py
View file @
39e23b21
...
...
@@ -37,10 +37,8 @@
import
nose.tools
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_format_object.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
numpy
import
nose.tools
import
numpy
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_format_simple.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
numpy
import
nose.tools
import
numpy
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_format_unsafe_cast.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
six
import
nose.tools
import
six
from
..dataformat
import
DataFormat
from
.
import
prefix
number42L
=
long
(
42
)
if
six
.
PY2
else
int
(
42
)
# noqa
...
...
beat/backend/python/test/test_format_valid_cast.py
View file @
39e23b21
...
...
@@ -34,14 +34,12 @@
###################################################################################
import
numpy
import
nose.tools
import
numpy
from
..dataformat
import
DataFormat
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_hash.py
View file @
39e23b21
...
...
@@ -39,7 +39,6 @@ import six
from
..
import
hash
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_helpers.py
View file @
39e23b21
...
...
@@ -37,22 +37,21 @@
# Tests for experiment execution
import
os
import
shutil
import
tempfile
import
unittest
import
zmq
import
tempfile
import
shutil
from
..algorithm
import
Algorithm
from
..database
import
Database
from
..execution
import
MessageHandler
from
..hash
import
hashDataset
from
..hash
import
toPath
from
..algorithm
import
Algorithm
from
..helpers
import
create_inputs_from_configuration
from
..helpers
import
AccessMode
from
..execution
import
MessageHandler
from
..helpers
import
create_inputs_from_configuration
from
.
import
prefix
# ----------------------------------------------------------
...
...
beat/backend/python/test/test_inputs.py
View file @
39e23b21
This diff is collapsed.
Click to expand it.
beat/backend/python/test/test_loop_executor.py
View file @
39e23b21
...
...
@@ -37,31 +37,28 @@
# Tests for experiment execution
import
os
import
unittest
import
zmq
import
tempfile
import
shutil
import
simplejson
import
numpy
as
np
import
tempfile
import
unittest
from
copy
import
deepcopy
from
..execution
import
AlgorithmExecutor
from
..execution
import
LoopExecutor
from
..execution
import
MessageHandler
from
..execution
import
LoopMessageHandler
from
..exceptions
import
RemoteException
import
numpy
as
np
import
simplejson
import
zmq
from
..algorithm
import
Algorithm
from
..dataformat
import
DataFormat
from
..data