Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
d829ac7c
Commit
d829ac7c
authored
Nov 16, 2020
by
Samuel GAIST
Browse files
[execution][docker] Complete refactor of database raw data sharing
parent
1a66c641
Pipeline
#45622
failed with stage
in 38 minutes and 22 seconds
Changes
12
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
beat/core/execution/docker.py
View file @
d829ac7c
...
...
@@ -384,14 +384,15 @@ class DockerExecutor(RemoteExecutor):
file_path
=
result
[
"path"
]
__add_writable_volume
(
file_path
)
def
__s
hare
_databases
(
self
,
algorithm_container
,
db_infos
):
"""Add volumes to the algorithm container f
or
the dataset
s
"""
def
__s
etup
_databases
_raw_access
(
self
,
algorithm_container
):
"""Add volumes to the algorithm container
i
f the data
ba
se
allows tha
t"""
for
database_name
,
database
in
self
.
databases
.
items
():
db_data
=
database
.
data
algorithm_container
.
add_volume
(
db_data
[
"root_folder"
],
os
.
path
.
join
(
"/databases"
,
database_name
)
)
if
db_data
.
get
(
"direct_rawdata_access"
,
False
):
algorithm_container
.
add_volume
(
db_data
[
"root_folder"
],
os
.
path
.
join
(
"/databases"
,
database_name
)
)
def
process
(
self
,
virtual_memory_in_megabytes
=
0
,
max_cpu_percent
=
0
,
timeout_in_minutes
=
0
...
...
@@ -481,8 +482,6 @@ class DockerExecutor(RemoteExecutor):
network_name
=
self
.
data
.
pop
(
"network_name"
,
"bridge"
)
databases_infos
=
{}
share_databases
=
self
.
data
.
pop
(
"share_databases"
,
False
)
if
len
(
self
.
databases
)
>
0
:
databases_infos
[
"db"
]
=
self
.
__create_db_container
(
datasets_uid
,
network_name
...
...
@@ -537,8 +536,7 @@ class DockerExecutor(RemoteExecutor):
loop_algorithm_container
,
volume_cache_mount_point
,
self
.
data
[
"loop"
]
)
if
share_databases
:
self
.
__share_databases
(
loop_algorithm_container
,
databases_infos
)
self
.
__setup_databases_raw_access
(
loop_algorithm_container
)
# Start the container
self
.
host
.
start
(
...
...
@@ -584,8 +582,7 @@ class DockerExecutor(RemoteExecutor):
algorithm_container
,
volume_cache_mount_point
,
self
.
data
)
if
share_databases
:
self
.
__share_databases
(
algorithm_container
,
databases_infos
)
self
.
__setup_databases_raw_access
(
algorithm_container
)
# Start the container
self
.
host
.
start
(
...
...
beat/core/test/prefix/algorithms/user/
shared_dataset
s/1.json
→
beat/core/test/prefix/algorithms/user/
integers_rawdata_acces
s/1.json
View file @
d829ac7c
{
"schema_version"
:
2
,
"schema_version"
:
3
,
"language"
:
"python"
,
"api_version"
:
2
,
"type"
:
"
autonomous
"
,
"type"
:
"
sequential
"
,
"splittable"
:
false
,
"parameters"
:
{
"sync"
:
{
"default"
:
"in1"
,
"type"
:
"string"
}
},
"groups"
:
[
{
"name"
:
"main"
,
"inputs"
:
{
"in1"
:
{
"type"
:
"user/single_integer/1"
},
"in2"
:
{
"in_data"
:
{
"type"
:
"user/single_integer/1"
}
},
"outputs"
:
{
"out"
:
{
"out
_data
"
:
{
"type"
:
"user/single_integer/1"
}
}
...
...
beat/core/test/prefix/algorithms/user/
shared_dataset
s/1.py
→
beat/core/test/prefix/algorithms/user/
integers_rawdata_acces
s/1.py
View file @
d829ac7c
...
...
@@ -31,33 +31,20 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import
os
import
numpy
class
Algorithm
:
def
__init__
(
self
):
self
.
offset
=
1
def
setup
(
self
,
parameters
):
self
.
sync
=
parameters
[
"sync"
]
return
True
def
process
(
self
,
data_loaders
,
outputs
):
data_loader
=
data_loaders
.
loaderOf
(
"in1"
)
print
(
os
.
listdir
(
"/databases/integers_db/1"
))
with
open
(
"/databases/integers_db/1/datafile.txt"
,
"rt"
)
as
shared_data
:
def
process
(
self
,
inputs
,
data_loaders
,
outputs
):
with
open
(
"/databases/simple_rawdata_access/1/datafile.txt"
,
"rt"
)
as
shared_data
:
value
=
shared_data
.
read
()
shared_offset
=
int
(
value
)
for
i
in
range
(
data_loader
.
count
(
self
.
sync
)):
view
=
data_loader
.
view
(
self
.
sync
,
i
)
(
data
,
start
,
end
)
=
view
[
view
.
count
()
-
1
]
value
=
numpy
.
int32
(
data
[
"in1"
].
value
+
data
[
"in2"
].
value
+
shared_offset
)
outputs
[
"out"
].
write
({
"value"
:
value
},
end
)
out_data
=
{
"value"
:
numpy
.
int32
(
inputs
[
"in_data"
].
data
.
value
+
shared_offset
)}
outputs
[
"out_data"
].
write
(
out_data
)
return
True
beat/core/test/prefix/databases/simple_rawdata_access/1.json
View file @
d829ac7c
{
"root_folder"
:
"/tmp/beat_core_test"
,
"environment"
:
{
"name"
:
"Example databases"
,
"version"
:
"1.4.0"
},
"direct_rawdata_access"
:
true
,
"protocols"
:
[
{
...
...
beat/core/test/prefix/databases/simple_rawdata_access/2.json
View file @
d829ac7c
{
"root_folder"
:
"/tmp/beat_core_test"
,
"environment"
:
{
"name"
:
"Example databases"
,
"version"
:
"1.4.0"
},
"direct_rawdata_access"
:
true
,
"protocols"
:
[
{
...
...
beat/core/test/prefix/experiments/errors/user/single/1/single_no_rawdata_access.json
0 → 100644
View file @
d829ac7c
{
"analyzers"
:
{
"analysis"
:
{
"algorithm"
:
"v1/integers_echo_analyzer/1"
,
"inputs"
:
{
"in_data"
:
"in"
}
}
},
"blocks"
:
{
"echo"
:
{
"algorithm"
:
"user/integers_rawdata_access/1"
,
"inputs"
:
{
"in_data"
:
"in"
},
"outputs"
:
{
"out_data"
:
"out"
}
}
},
"datasets"
:
{
"set"
:
{
"database"
:
"simple/1"
,
"protocol"
:
"protocol"
,
"set"
:
"set"
}
},
"globals"
:
{
"queue"
:
"queue"
,
"environment"
:
{
"name"
:
"Python for tests"
,
"version"
:
"1.3.0"
}
}
}
beat/core/test/prefix/experiments/errors/user/single/1/single_no_rawdata_access.rst
0 → 100644
View file @
d829ac7c
Test experiment for validating the rawdata access feature. This ensure that if the
database does not have the field then nothing is mounted.
beat/core/test/prefix/experiments/user/user/integers_addition/1/shared_datasets.json
deleted
100644 → 0
View file @
1a66c641
{
"blocks"
:
{
"addition"
:
{
"algorithm"
:
"user/shared_datasets/1"
,
"inputs"
:
{
"in1"
:
"a"
,
"in2"
:
"b"
},
"outputs"
:
{
"out"
:
"sum"
}
}
},
"datasets"
:
{
"integers"
:
{
"database"
:
"integers_db/1"
,
"protocol"
:
"double"
,
"set"
:
"double"
}
},
"analyzers"
:
{
"analysis"
:
{
"algorithm"
:
"v1/integers_analysis/1"
,
"inputs"
:
{
"input"
:
"input"
}
}
},
"globals"
:
{
"environment"
:
{
"name"
:
"Python for tests"
,
"version"
:
"1.3.0"
},
"queue"
:
"queue"
,
"user/shared_datasets/1"
:
{
"sync"
:
"in1"
}
}
}
beat/core/test/prefix/experiments/user/user/integers_addition/1/shared_datasets.rst
deleted
100644 → 0
View file @
1a66c641
This experiment is to test whether the dataset sharing
functionnality works as expected.
beat/core/test/prefix/experiments/user/user/single/1/single_rawdata_access.json
0 → 100644
View file @
d829ac7c
{
"analyzers"
:
{
"analysis"
:
{
"algorithm"
:
"v1/integers_echo_analyzer/1"
,
"inputs"
:
{
"in_data"
:
"in"
}
}
},
"blocks"
:
{
"echo"
:
{
"algorithm"
:
"user/integers_rawdata_access/1"
,
"inputs"
:
{
"in_data"
:
"in"
},
"outputs"
:
{
"out_data"
:
"out"
}
}
},
"datasets"
:
{
"set"
:
{
"database"
:
"simple_rawdata_access/1"
,
"protocol"
:
"protocol"
,
"set"
:
"set"
}
},
"globals"
:
{
"queue"
:
"queue"
,
"environment"
:
{
"name"
:
"Python for tests"
,
"version"
:
"1.3.0"
}
}
}
beat/core/test/prefix/experiments/user/user/single/1/single_rawdata_access.rst
0 → 100644
View file @
d829ac7c
Test experiment for validating the rawdata access feature
beat/core/test/test_docker_execution.py
View file @
d829ac7c
...
...
@@ -63,6 +63,21 @@ BUILDER_IMAGE = (
# ----------------------------------------------------------
def
write_rawdata_for_database
(
database_name
,
raw_data
):
"""Generate raw data for give database"""
db
=
Database
(
test_prefix
,
database_name
)
nose
.
tools
.
assert_true
(
db
.
valid
,
db
.
errors
)
data_sharing_path
=
db
.
data
[
"root_folder"
]
with
open
(
os
.
path
.
join
(
data_sharing_path
,
"datafile.txt"
),
"wt"
)
as
data_file
:
data_file
.
write
(
"{}"
.
format
(
raw_data
))
# ----------------------------------------------------------
class
TestDockerExecution
(
BaseExecutionMixIn
):
@
classmethod
def
setup_class
(
cls
):
...
...
@@ -152,24 +167,26 @@ class TestDockerExecution(BaseExecutionMixIn):
nose
.
tools
.
assert_is_none
(
result
)
@
slow
def
test_databases_sharing
(
self
):
db
=
Database
(
test_prefix
,
"integers_db/1"
)
nose
.
tools
.
assert_true
(
db
.
valid
,
db
.
errors
)
data_sharing_path
=
db
.
data
[
"root_folder"
]
def
test_database_rawdata_access
(
self
):
offset
=
12
with
open
(
os
.
path
.
join
(
data_sharing_path
,
"datafile.txt"
),
"wt"
)
as
data_file
:
data_file
.
write
(
"{}"
.
format
(
offset
)
)
write_rawdata_for_database
(
"simple_rawdata_access/1"
,
offset
)
result
=
self
.
execute
(
"user/user/integers_addition/1/shared_datasets"
,
[{
"sum"
:
495
+
9
*
offset
,
"nb"
:
9
}],
share_databases
=
True
,
"user/user/single/1/single_rawdata_access"
,
[{
"out_data"
:
42
+
offset
}]
)
nose
.
tools
.
assert_is_none
(
result
)
@
slow
def
test_database_no_rawdata_access
(
self
):
write_rawdata_for_database
(
"simple/1"
,
"should not be loaded"
)
result
=
self
.
execute
(
"errors/user/single/1/single_no_rawdata_access"
,
[
None
])
nose
.
tools
.
eq_
(
result
[
"status"
],
1
)
nose
.
tools
.
assert_true
(
"FileNotFoundError"
in
result
[
"user_error"
])
@
slow
def
test_single_1_prepare_error
(
self
):
result
=
self
.
execute
(
"errors/user/single/1/prepare_error"
,
[
None
])
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment