...
 
Commits (39)
......@@ -247,3 +247,12 @@ class Database(BackendDatabase):
if self.schema_version != 1:
for view in protocol["views"].keys():
self._validate_view(view)
@property
def is_database_rawdata_access_enabled(self):
"""Returns whether raw data sharing was enabled
This property is only useful for the Docker executor.
"""
return self.data.get("direct_rawdata_access", False)
......@@ -795,6 +795,17 @@ class Container:
self._entrypoint = None
self._temporary_filesystems = {"/tmp": "500k", "/run": "500k"} # nosec
client = docker.from_env()
docker_image = client.images.get(image)
custom_tmpfs = docker_image.labels.get("beat.env.custom_tmpfs")
if custom_tmpfs is not None:
# import ipdb; ipdb.set_trace()
custom_tmpfs = json.loads(custom_tmpfs)
for path, size in custom_tmpfs.items():
self._temporary_filesystems[path] = size
def set_name(self, name):
""" Set the name to be used by the container in place of the docker
auto generated one.
......
......@@ -384,6 +384,16 @@ class DockerExecutor(RemoteExecutor):
file_path = result["path"]
__add_writable_volume(file_path)
def __setup_databases_raw_access(self, algorithm_container):
"""Add volumes to the algorithm container if the database allows that"""
for database_name, database in self.databases.items():
db_data = database.data
if db_data.get("direct_rawdata_access", False):
algorithm_container.add_volume(
db_data["root_folder"], os.path.join("/databases", database_name)
)
def process(
self, virtual_memory_in_megabytes=0, max_cpu_percent=0, timeout_in_minutes=0
):
......@@ -526,6 +536,8 @@ class DockerExecutor(RemoteExecutor):
loop_algorithm_container, volume_cache_mount_point, self.data["loop"]
)
self.__setup_databases_raw_access(loop_algorithm_container)
# Start the container
self.host.start(
loop_algorithm_container,
......@@ -570,6 +582,8 @@ class DockerExecutor(RemoteExecutor):
algorithm_container, volume_cache_mount_point, self.data
)
self.__setup_databases_raw_access(algorithm_container)
# Start the container
self.host.start(
algorithm_container,
......
......@@ -153,13 +153,6 @@ class Experiment(object):
Attributes:
label (str): The full, valid label of this experiment
description (str): The short description string, loaded from the JSON
file if one was set.
documentation (str): The full-length docstring for this object.
storage (object): A simple object that provides information about file
paths for this toolchain
......@@ -1063,7 +1056,7 @@ class Experiment(object):
@property
def label(self):
"""Returns the label of this experiment"""
"""Label of this experiment"""
return self._label or "__unlabelled_experiment__"
......@@ -1076,7 +1069,7 @@ class Experiment(object):
@property
def schema_version(self):
"""Returns the schema version"""
"""Schema version"""
return self.data.get("schema_version", 1)
......
......@@ -150,13 +150,6 @@ class Plotter(object):
Attributes:
name (str): The algorithm name
description (str): The short description string, loaded from the JSON
file if one was set.
documentation (str): The full-length docstring for this object.
storage (object): A simple object that provides information about file
paths for this algorithm
......@@ -166,12 +159,6 @@ class Plotter(object):
libraries (dict): A mapping object defining other libraries this plotter
needs to load so it can work properly.
uses (dict): A mapping object defining the required library import name
(keys) and the full-names (values).
parameters (dict): A dictionary containing all pre-defined parameters
that this algorithm accepts.
errors (list): A list containing errors found while loading this
algorithm.
......@@ -317,7 +304,7 @@ class Plotter(object):
@property
def name(self):
"""Returns the name of this object
"""The name of this object
"""
return self._name or "__unnamed_plotter__"
......@@ -342,7 +329,9 @@ class Plotter(object):
return self.data.get("api_version", 1)
def uses_dict(self):
"""Returns the usage dictionary for all dependent modules"""
"""A mapping object defining the required library import name (keys) and the
full-names (values).
"""
if self.data["language"] == "unknown":
raise RuntimeError("plotter has no programming language set")
......
......@@ -19,23 +19,13 @@
"items": { "$ref": "#/definitions/protocol" }
},
"environment": {
"type": "object",
"properties": {
"name": { "type": "string" },
"version": { "type": "string" }
},
"required": [
"name",
"version"
],
"additionalProperties": false
},
"environment": {"$ref": "common.json#/definitions/environment"},
"description": { "$ref": "../common/1.json#/definitions/description" },
"schema_version": { "$ref": "../common/1.json#/definitions/version" }
"schema_version": { "$ref": "../common/1.json#/definitions/version" },
"direct_rawdata_access": {"$ref": "common.json#/definitions/direct_rawdata_access"}
},
"required": [
......
......@@ -21,20 +21,11 @@
"description": { "$ref": "../common/1.json#/definitions/description" },
"environment": {
"type": "object",
"properties": {
"name": { "type": "string" },
"version": { "type": "string" }
},
"required": [
"name",
"version"
],
"additionalProperties": false
},
"environment": {"$ref": "common.json#/definitions/environment"},
"schema_version": { "const": 2 },
"schema_version": { "const": 2 }
"direct_rawdata_access": {"$ref": "common.json#/definitions/direct_rawdata_access"}
},
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Algorithm common components descriptor",
"description": "This schema defines the components used in one or more versions of the Database",
"definitions": {
"direct_rawdata_access": {
"type": "boolean",
"default": false
},
"environment": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
}
},
"required": [
"name",
"version"
],
"additionalProperties": false
}
}
}
......@@ -41,7 +41,7 @@
"environment": { "$ref": "#/definitions/environment" }
},
"patternProperties": {
"^[a-zA-Z_][a-zA-Z0-9_]*/[a-zA-Z_][a-zA-Z0-9_-]*/[0-9]+$": {
"^[a-zA-Z_][a-zA-Z0-9_-]*/[a-zA-Z_][a-zA-Z0-9_-]*/[0-9]+$": {
"$ref": "#/definitions/parameter_set"
}
},
......
......@@ -74,8 +74,6 @@ class Statistics(object):
errors (list): A list strings containing errors found while loading this
statistics information.
data (dict): The original data for these statistics
"""
def __init__(self, data=None):
......
......@@ -35,6 +35,7 @@
# Basic setup for slow tests
import json
import logging
import os
import shutil
......@@ -84,14 +85,46 @@ if VERBOSE_TEST_LOGGING:
logger.addHandler(handler)
def sync_prefixes(source_prefixes, target_prefix):
for path in source_prefixes:
sp.check_call(["rsync", "-arz", path, target_prefix])
def initialize_db_root_folder(database_root_folder, databases_path):
os.makedirs(database_root_folder, exist_ok=True)
for root, dirs, files in os.walk(databases_path, topdown=False):
for file_ in files:
if file_.endswith(".json"):
path = os.path.join(root, file_)
try:
with open(path, "rt") as db_file:
declaration = json.load(db_file)
except json.JSONDecodeError:
# some are explicitly invalid.
continue
else:
declaration["root_folder"] = database_root_folder
with open(path, "wt") as db_file:
json.dump(declaration, db_file, indent=4)
def setup_root_db_folder():
initialize_db_root_folder(
os.path.join(prefix_folder, "beat_core_test"), os.path.join(prefix, "databases")
)
def setup_package():
prefixes = [
sync_prefixes(
[
pkg_resources.resource_filename("beat.backend.python.test", "prefix"),
pkg_resources.resource_filename("beat.core.test", "prefix"),
]
],
prefix_folder,
)
for path in prefixes:
sp.check_call(["rsync", "-arz", path, prefix_folder])
setup_root_db_folder()
if DOCKER_NETWORK_TEST_ENABLED:
import docker
......
{
"schema_version": 3,
"language": "python",
"api_version": 2,
"type": "autonomous_loop_processor",
"splittable": false,
"groups": [
{
"inputs": {
"in": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
},
"loop": {
"request": {
"type": "user/1d_array_of_integers/1"
},
"answer": {
"type": "user/single_float/1"
}
}
}
],
"parameters": {
"step": {
"default": 1,
"type": "int8",
"description": "Counter step size"
}
}
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import numpy as np
class Algorithm:
def __init__(self):
self.step = 1
def setup(self, parameters):
self.step = parameters["step"]
return True
def process(self, data_loaders, outputs, loop_channel):
cnt = 1
is_valid, delta = loop_channel.validate({"value": np.full(10, cnt)})
while not is_valid:
cnt = cnt + self.step
is_valid, delta = loop_channel.validate({"value": np.full(10, cnt)})
delta = delta.value
data_loader = data_loaders.loaderOf("in")
for i in range(data_loader.count()):
view = data_loader.view("in", i)
(data, _, end) = view[view.count() - 1]
value = data["in"].value
new_value = (value + cnt) * delta
outputs["out"].write({"value": np.int32(new_value)}, end)
return True
{
"schema_version": 3,
"language": "python",
"api_version": 2,
"type": "sequential",
"splittable": false,
"groups": [
{
"name": "main",
"inputs": {
"in_data": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out_data": {
"type": "user/single_integer/1"
}
}
}
]
}
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2020 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import numpy
class Algorithm:
def process(self, inputs, data_loaders, outputs):
with open(
"/databases/simple_rawdata_access/1/datafile.txt", "rt"
) as shared_data:
value = shared_data.read()
shared_offset = int(value)
out_data = {"value": numpy.int32(inputs["in_data"].data.value + shared_offset)}
outputs["out_data"].write(out_data)
return True
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
]
}
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Does not exist",
"version": "1.4.0"
......
......@@ -4,7 +4,7 @@
"name": "Does not exist",
"version": "1.4.0"
},
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "large",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "large",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"sets": [
......
{
"root_folder": "/tmp/path/not/set"
"root_folder": "/tmp/beat_core_test"
}
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "protocol",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/path/not/set",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "protocol",
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"protocols": [
{
"name": "protocol",
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Example databases",
"version": "1.4.0"
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Example databases",
"version": "1.4.0"
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Not existing",
"version": "1.4.0"
......
{
"root_folder": "/tmp/foo/bar",
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Not existing",
"version": "1.4.0"
......
{
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Example databases",
"version": "1.4.0"
},
"direct_rawdata_access": true,
"protocols": [
{
"name": "protocol",
"template": "test_integers",
"sets": [
{
"name": "set",
"template": "set",
"view": "View",
"outputs": {
"out": "user/single_integer/1"
}
},
{
"name": "set2",
"template": "set",
"view": "View2",
"outputs": {
"out": "user/single_integer/1"
}
}
]
},
{
"name": "protocol2",
"template": "test_integers",
"sets": [
{
"name": "set",
"template": "set",
"view": "LargeView",
"outputs": {
"out": "user/single_integer/1"
}
},
{
"name": "set2",
"template": "set",
"view": "View2",
"outputs": {
"out": "user/single_integer/1"
}
}
]
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
from collections import namedtuple
import numpy
from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
):
"""Initializes the database"""
super().setup(root_folder, outputs, parameters, start_index, end_index)
return True
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(42)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
# ----------------------------------------------------------
class View2(BaseView):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(53)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
# ----------------------------------------------------------
class LargeView(BaseView):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(0), Entry(1), Entry(2), Entry(3), Entry(4)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
.. Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.backend.python module of the BEAT platform. ..
.. ..
.. Redistribution and use in source and binary forms, with or without
.. modification, are permitted provided that the following conditions are met:
.. 1. Redistributions of source code must retain the above copyright notice, this
.. list of conditions and the following disclaimer.
.. 2. Redistributions in binary form must reproduce the above copyright notice,
.. this list of conditions and the following disclaimer in the documentation
.. and/or other materials provided with the distribution.
.. 3. Neither the name of the copyright holder nor the names of its contributors
.. may be used to endorse or promote products derived from this software without
.. specific prior written permission.
.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
.. ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
.. WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
.. DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
.. FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
.. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
.. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
.. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The Simple Digit Database
-------------------------
This database emits two integers (one from each of its set). The first integer
(from ``set``) has a value of 42, while the second (from ``set2``), a value of
53.
{
"root_folder": "/tmp/beat_core_test",
"environment": {
"name": "Example databases",
"version": "1.4.0"
},
"direct_rawdata_access": true,
"protocols": [
{
"name": "protocol",
"template": "protocol/1",
"views": {
"set": {
"view": "View"
},
"set2": {
"view": "View2"
}
}
},
{
"name": "protocol2",
"template": "protocol2/1",
"views": {
"set": {
"view": "LargeView"
},
"set2": {
"view": "View2"
}
}
}
],
"schema_version": 2
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
from collections import namedtuple
import numpy
from beat.backend.python.database import View as BaseView
class View(BaseView):
def setup(
self, root_folder, outputs, parameters, start_index=None, end_index=None,
):
"""Initializes the database"""
super().setup(root_folder, outputs, parameters, start_index, end_index)
return True
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(42)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
# ----------------------------------------------------------
class View2(BaseView):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(53)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
# ----------------------------------------------------------
class LargeView(BaseView):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(0), Entry(1), Entry(2), Entry(3), Entry(4)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
{
"analyzers": {
"analysis": {
"algorithm": "v1/integers_echo_analyzer/1",
"inputs": {
"in_data": "in"
}
}
},
"blocks": {
"echo": {
"algorithm": "user/integers_rawdata_access/1",
"inputs": {
"in_data": "in"
},
"outputs": {
"out_data": "out"
}
}
},
"datasets": {
"set": {
"database": "simple/1",
"protocol": "protocol",
"set": "set"
}
},
"globals": {
"queue": "queue",
"environment": {
"name": "Python for tests",
"version": "1.3.0"
}
}
}
Test experiment for validating the rawdata access feature. This ensure that if the
database does not have the field then nothing is mounted.
{
"schema_version": 2,
"blocks": {
},
"loops": {
"loop_super_block": {
"processor_algorithm": "user-with_dash/block-with_dash/1",
"processor_parameters": {
"step": 1
},
"processor_inputs": {
"in": "in"
},
"processor_outputs": {
"out": "out"
},
"evaluator_algorithm": "user/db_input_loop_evaluator/1",
"evaluator_parameters": {
"threshold": 9
},
"evaluator_inputs": {
"in_loop": "in_loop"
},
"evaluator_outputs": {
"out_loop": "out_loop"
}
}
},
"analyzers": {
"analysis": {
"algorithm": "v1/integers_analysis/1",
"parameters": {
},
"inputs": {
"input": "input"
}
},
"loop_analysis": {
"algorithm": "v1/integers_analysis/1",
"parameters": {
},
"inputs": {
"input": "input"
}
}
},
"datasets": {
"integers": {
"database": "integers_db/1",
"protocol": "double",
"set": "double"
},
"integers2": {
"database": "simple/1",
"protocol": "protocol2",
"set": "set"
}
},
"globals": {
"environment": {
"name": "Python for tests",
"version": "1.3.0"
},
"queue": "queue",
"user-with_dash/block-with_dash/1": {
"step": 2
},
"user/db_input_loop_evaluator/1" : {
"threshold": 8
}
}
}
Test that loading an experiment with an algorithm containg dashes
in its author name works.
See beat/beat.core#104
{
"analyzers": {
"analysis": {
"algorithm": "v1/integers_echo_analyzer/1",
"inputs": {
"in_data": "in"
}
}
},
"blocks": {
"echo": {
"algorithm": "user/integers_rawdata_access/1",
"inputs": {
"in_data": "in"
},
"outputs": {
"out_data": "out"
}
}
},
"datasets": {
"set": {
"database": "simple_rawdata_access/1",
"protocol": "protocol",
"set": "set"
}
},
"globals": {
"queue": "queue",
"environment": {
"name": "Python for tests",
"version": "1.3.0"
}
}
}
Test experiment for validating the rawdata access feature
......@@ -47,6 +47,7 @@ def test_export():
yield export, f"integers_db/{i}"
yield export, f"simple/{i}"
yield export, f"large/{i}"
yield export, f"simple_rawdata_access/{i}"
@nose.tools.with_setup(teardown=cleanup)
......@@ -60,3 +61,22 @@ def export(db_name):
# load from tmp_prefix and validates
exported = Database(tmp_prefix, db_name)
nose.tools.assert_true(exported.valid, "\n * %s" % "\n * ".join(exported.errors))
def test_rawdata_access():
for i in range(1, 3):
yield rawdata_access, f"integers_db/{i}", False
yield rawdata_access, f"simple/{i}", False
yield rawdata_access, f"large/{i}", False
yield rawdata_access, f"simple_rawdata_access/{i}", True
@nose.tools.with_setup(teardown=cleanup)
def rawdata_access(db_name, rawdata_access_enabled):
obj = Database(prefix, db_name)
nose.tools.assert_true(obj.valid, "\n * %s" % "\n * ".join(obj.errors))
nose.tools.assert_equal(
obj.is_database_rawdata_access_enabled, rawdata_access_enabled
)
......@@ -107,8 +107,7 @@ class NetworkTest(NoDiscoveryTests):
class UserTest(NoDiscoveryTests):
@slow
def test_user(self):
"""Test that the uid property is correctly used.
"""
"""Test that the uid property is correctly used."""
container = self.host.create_container("debian:8.4", ["id"])
container.uid = 10000
......@@ -127,8 +126,7 @@ class UserTest(NoDiscoveryTests):
class EnvironmentVariableTest(NoDiscoveryTests):
@slow
def test_environment_variable(self):
"""Test that the uid property is correctly used.
"""
"""Test that the uid property is correctly used."""
container = self.host.create_container("debian:8.4", ["env"])
container.add_environment_variable("DOCKER_TEST", "good")
......@@ -143,8 +141,7 @@ class EnvironmentVariableTest(NoDiscoveryTests):
class WorkdirTest(NoDiscoveryTests):
@slow
def test_workdir(self):
"""Test that the workdir property is correctly used.
"""
"""Test that the workdir property is correctly used."""
with TemporaryDirectory() as tmp_folder:
test_file = "test.txt"
......@@ -169,8 +166,7 @@ class WorkdirTest(NoDiscoveryTests):
class EntrypointTest(NoDiscoveryTests):
@slow
def test_entrypoint(self):
"""Test that the entrypoint property is correctly used.
"""
"""Test that the entrypoint property is correctly used."""
container = self.host.create_container("debian:8.4", ["42"])
container.set_entrypoint("echo")
......@@ -187,8 +183,7 @@ class EntrypointTest(NoDiscoveryTests):
class TmpfsTest(NoDiscoveryTests):
def test_tmpfs(self):
"""Test that the tmpfs are properly mounted and usable.
"""
"""Test that the tmpfs are properly mounted and usable."""
container = self.host.create_container(
"debian:8.4", ["touch", "/dummy/test.txt"]
......@@ -213,33 +208,6 @@ class TmpfsTest(NoDiscoveryTests):
self.assertEqual(status, 0)
self.assertEqual(logs, "")
def test_tmpfs_size(self):
"""Test that the tmpfs are respected.
"""
container = self.host.create_container(
"debian:8.4", ["dd", "if=/dev/zero", "of=/dummy/test.txt"]
)
tmpfs_list = container.temporary_filesystems
self.assertEqual(len(tmpfs_list), 2)
container.add_tmpfs("/dummy", "1M")
tmpfs_list = container.temporary_filesystems
self.assertEqual(len(tmpfs_list), 3)
self.host.start(container)
status = self.host.wait(container)
logs = self.host.logs(container)
if status != 0:
print(logs)
self.assertEqual(status, 1)
self.assertTrue("No space left" in logs)
class AsyncTest(NoDiscoveryTests):
@slow
......@@ -300,7 +268,7 @@ class AsyncTest(NoDiscoveryTests):
self.assertEqual(self.host.logs(container), "")
class AsyncWithEnvironmentTest(unittest.TestCase):
class WithDiscoveryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.host = Host(raise_on_errors=False)
......@@ -314,6 +282,31 @@ class AsyncWithEnvironmentTest(unittest.TestCase):
self.host.teardown()
self.assertFalse(self.host.containers) # All containers are gone
class TmpfsWithEnvironmentTest(WithDiscoveryTests):
def test_tmpfs_from_label(self):
"""Test that the tmpfs are respected."""
container = self.host.create_container(
"Python for tests (1.3.0)",
["dd", "if=/dev/zero", "of=/custom_tmpfs/test.txt"],
)
tmpfs_list = container.temporary_filesystems
self.assertEqual(len(tmpfs_list), 3)
print(tmpfs_list)
self.host.start(container)
status = self.host.wait(container)
logs = self.host.logs(container)
if status != 0:
print(logs)
self.assertEqual(status, 1)
self.assertTrue("No space left" in logs)
class AsyncWithEnvironmentTest(WithDiscoveryTests):
@slow
def test_memory_limit(self):
......
......@@ -41,11 +41,15 @@ import subprocess # nosec
import nose.tools
from beat.core.database import Database
from ..dock import Host
from ..execution import DockerExecutor
from . import DOCKER_NETWORK_TEST_ENABLED
from . import network_name
from . import prefix as test_prefix
from . import prefix_folder
from . import setup_root_db_folder
from .test_execution import BaseExecutionMixIn
from .utils import DOCKER_TEST_IMAGES
from .utils import cleanup
......@@ -60,6 +64,21 @@ BUILDER_IMAGE = (
# ----------------------------------------------------------
def write_rawdata_for_database(database_name, raw_data):
"""Generate raw data for give database"""
db = Database(test_prefix, database_name)
nose.tools.assert_true(db.valid, db.errors)
data_sharing_path = db.data["root_folder"]
with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
data_file.write("{}".format(raw_data))
# ----------------------------------------------------------
class TestDockerExecution(BaseExecutionMixIn):
@classmethod
def setup_class(cls):
......@@ -127,6 +146,8 @@ class TestDockerExecution(BaseExecutionMixIn):
]
)
setup_root_db_folder()
@slow
@skipif(not DOCKER_NETWORK_TEST_ENABLED, "Network test disabled")
def test_custom_network(self):
......@@ -148,6 +169,27 @@ class TestDockerExecution(BaseExecutionMixIn):
nose.tools.assert_is_none(result)
@slow
def test_database_rawdata_access(self):
offset = 12
write_rawdata_for_database("simple_rawdata_access/1", offset)
result = self.execute(
"user/user/single/1/single_rawdata_access", [{"out_data": 42 + offset}]
)
nose.tools.assert_is_none(result)
@slow
def test_database_no_rawdata_access(self):
write_rawdata_for_database("simple/1", "should not be loaded")
result = self.execute("errors/user/single/1/single_no_rawdata_access", [None])
nose.tools.eq_(result["status"], 1)
nose.tools.assert_true("FileNotFoundError" in result["user_error"])
@slow
def test_single_1_prepare_error(self):