Commit 5c42cca1 authored by Samuel GAIST's avatar Samuel GAIST

[test][docker_execution] Implement test for database sharing

parent 6e28d902
Pipeline #44868 failed with stage
in 25 minutes and 15 seconds
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"parameters": {
"sync": {
"default": "in1",
"type": "string"
}
},
"groups": [
{
"inputs": {
"in1": {
"type": "user/single_integer/1"
},
"in2": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
}
]
}
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2020 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import os
import numpy
class Algorithm:
def __init__(self):
self.offset = 1
def setup(self, parameters):
self.sync = parameters["sync"]
return True
def process(self, data_loaders, outputs):
data_loader = data_loaders.loaderOf("in1")
print(os.listdir("/databases/integers_db/1"))
with open("/databases/integers_db/1/datafile.txt", "rt") as shared_data:
value = shared_data.read()
shared_offset = int(value)
for i in range(data_loader.count(self.sync)):
view = data_loader.view(self.sync, i)
(data, start, end) = view[view.count() - 1]
value = numpy.int32(data["in1"].value + data["in2"].value + shared_offset)
outputs["out"].write({"value": value}, end)
return True
{
"blocks": {
"addition": {
"algorithm": "user/shared_datasets/1",
"parameters": {
},
"inputs": {
"in1": "a",
"in2": "b"
},
"outputs": {
"out": "sum"
}
}
},
"datasets": {
"integers": {
"database": "integers_db/1",
"protocol": "double",
"set": "double"
}
},
"analyzers": {
"analysis": {
"algorithm": "v1/integers_analysis/1",
"parameters": {
},
"inputs": {
"input": "input"
}
}
},
"globals": {
"environment": {
"name": "Python for tests",
"version": "1.3.0"
},
"queue": "queue"
}
}
This experiment is to test whether the dataset sharing
functionnality works as expected.
......@@ -37,9 +37,13 @@
# Tests for experiment execution within Docker containers
import os
import shutil
import subprocess # nosec
import nose.tools
import pkg_resources
from beat.core.database import Database
from ..dock import Host
from ..execution import DockerExecutor
......@@ -148,6 +152,29 @@ class TestDockerExecution(BaseExecutionMixIn):
nose.tools.assert_is_none(result)
@slow
def test_databases_sharing(self):
prefix = pkg_resources.resource_filename("beat.backend.python.test", "prefix")
db = Database(prefix, "integers_db/1")
nose.tools.assert_true(db.valid, db.errors)
data_sharing_path = db.data["root_folder"]
os.makedirs(data_sharing_path, exist_ok=True)
offset = 12
with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
data_file.write("{}".format(offset))
result = self.execute(
"user/user/integers_addition/1/shared_datasets",
[{"sum": 495 + 9 * offset, "nb": 9}],
share_databases=True,
)
shutil.rmtree(data_sharing_path)
nose.tools.assert_is_none(result)
@slow
def test_single_1_prepare_error(self):
result = self.execute("errors/user/single/1/prepare_error", [None])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment