beat/core/database.py

@@ -247,3 +247,12 @@ class Database(BackendDatabase):
             if self.schema_version != 1:
                 for view in protocol["views"].keys():
                     self._validate_view(view)
+
+    @property
+    def is_database_rawdata_access_enabled(self):
+        """Returns whether raw data sharing was enabled
+
+        This property is only useful for the Docker executor.
+        """
+
+        return self.data.get("direct_rawdata_access", False)

beat/core/dock.py

@@ -795,6 +795,17 @@ class Container:
         self._entrypoint = None
         self._temporary_filesystems = {"/tmp": "500k", "/run": "500k"}  # nosec
 
+        client = docker.from_env()
+        docker_image = client.images.get(image)
+        custom_tmpfs = docker_image.labels.get("beat.env.custom_tmpfs")
+
+        if custom_tmpfs is not None:
+            # import ipdb; ipdb.set_trace()
+            custom_tmpfs = json.loads(custom_tmpfs)
+
+            for path, size in custom_tmpfs.items():
+                self._temporary_filesystems[path] = size
+
     def set_name(self, name):
         """ Set the name to be used by the container in place of the docker
         auto generated one.

beat/core/execution/docker.py

@@ -384,6 +384,16 @@ class DockerExecutor(RemoteExecutor):
             file_path = result["path"]
             __add_writable_volume(file_path)
 
+    def __setup_databases_raw_access(self, algorithm_container):
+        """Add volumes to the algorithm container if the database allows that"""
+
+        for database_name, database in self.databases.items():
+            db_data = database.data
+            if db_data.get("direct_rawdata_access", False):
+                algorithm_container.add_volume(
+                    db_data["root_folder"], os.path.join("/databases", database_name)
+                )
+
     def process(
         self, virtual_memory_in_megabytes=0, max_cpu_percent=0, timeout_in_minutes=0
     ):
@@ -526,6 +536,8 @@ class DockerExecutor(RemoteExecutor):
                 loop_algorithm_container, volume_cache_mount_point, self.data["loop"]
             )
 
+            self.__setup_databases_raw_access(loop_algorithm_container)
+
             # Start the container
             self.host.start(
                 loop_algorithm_container,
@@ -570,6 +582,8 @@ class DockerExecutor(RemoteExecutor):
             algorithm_container, volume_cache_mount_point, self.data
         )
 
+        self.__setup_databases_raw_access(algorithm_container)
+
         # Start the container
         self.host.start(
             algorithm_container,

beat/core/experiment.py

@@ -153,13 +153,6 @@ class Experiment(object):
 
     Attributes:
 
-      label (str): The full, valid label of this experiment
-
-      description (str): The short description string, loaded from the JSON
-        file if one was set.
-
-      documentation (str): The full-length docstring for this object.
-
       storage (object): A simple object that provides information about file
         paths for this toolchain
 
@@ -1063,7 +1056,7 @@ class Experiment(object):
 
     @property
     def label(self):
-        """Returns the label of this experiment"""
+        """Label of this experiment"""
 
         return self._label or "__unlabelled_experiment__"
 
@@ -1076,7 +1069,7 @@ class Experiment(object):
 
     @property
     def schema_version(self):
-        """Returns the schema version"""
+        """Schema version"""
 
         return self.data.get("schema_version", 1)
 

beat/core/plotter.py

@@ -150,13 +150,6 @@ class Plotter(object):
 
     Attributes:
 
-      name (str): The algorithm name
-
-      description (str): The short description string, loaded from the JSON
-        file if one was set.
-
-      documentation (str): The full-length docstring for this object.
-
       storage (object): A simple object that provides information about file
         paths for this algorithm
 
@@ -166,12 +159,6 @@ class Plotter(object):
       libraries (dict): A mapping object defining other libraries this plotter
         needs to load so it can work properly.
 
-      uses (dict): A mapping object defining the required library import name
-        (keys) and the full-names (values).
-
-      parameters (dict): A dictionary containing all pre-defined parameters
-        that this algorithm accepts.
-
       errors (list): A list containing errors found while loading this
         algorithm.
 
@@ -317,7 +304,7 @@ class Plotter(object):
 
     @property
     def name(self):
-        """Returns the name of this object
+        """The name of this object
         """
         return self._name or "__unnamed_plotter__"
 
@@ -342,7 +329,9 @@ class Plotter(object):
         return self.data.get("api_version", 1)
 
     def uses_dict(self):
-        """Returns the usage dictionary for all dependent modules"""
+        """A mapping object defining the required library import name (keys) and the
+        full-names (values).
+        """
 
         if self.data["language"] == "unknown":
             raise RuntimeError("plotter has no programming language set")

beat/core/schema/database/1.json

@@ -19,23 +19,13 @@
       "items": { "$ref": "#/definitions/protocol" }
     },
 
-    "environment": {
-      "type": "object",
-      "properties": {
-        "name": { "type": "string" },
-        "version": { "type": "string" }
-      },
-      "required": [
-        "name",
-        "version"
-      ],
-      "additionalProperties": false
-    },
+    "environment": {"$ref": "common.json#/definitions/environment"},
 
     "description": { "$ref": "../common/1.json#/definitions/description" },
 
-    "schema_version": { "$ref": "../common/1.json#/definitions/version" }
+    "schema_version": { "$ref": "../common/1.json#/definitions/version" },
 
+    "direct_rawdata_access": {"$ref": "common.json#/definitions/direct_rawdata_access"}
   },
 
   "required": [

beat/core/schema/database/2.json

@@ -21,20 +21,11 @@
 
     "description": { "$ref": "../common/1.json#/definitions/description" },
 
-    "environment": {
-      "type": "object",
-      "properties": {
-        "name": { "type": "string" },
-        "version": { "type": "string" }
-      },
-      "required": [
-        "name",
-        "version"
-      ],
-      "additionalProperties": false
-    },
+    "environment": {"$ref": "common.json#/definitions/environment"},
+
+    "schema_version": { "const": 2 },
 
-    "schema_version": { "const": 2 }
+    "direct_rawdata_access": {"$ref": "common.json#/definitions/direct_rawdata_access"}
 
   },
 

beat/core/schema/database/common.json

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "Algorithm common components descriptor",
+    "description": "This schema defines the components used in one or more versions of the Database",
+    "definitions": {
+        "direct_rawdata_access": {
+            "type": "boolean",
+            "default": false
+        },
+        "environment": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "version": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "name",
+                "version"
+            ],
+            "additionalProperties": false
+        }
+    }
+}

beat/core/schema/experiment/common.json

@@ -41,7 +41,7 @@
         "environment":  { "$ref": "#/definitions/environment" }
       },
       "patternProperties": {
-        "^[a-zA-Z_][a-zA-Z0-9_]*/[a-zA-Z_][a-zA-Z0-9_-]*/[0-9]+$": {
+        "^[a-zA-Z_][a-zA-Z0-9_-]*/[a-zA-Z_][a-zA-Z0-9_-]*/[0-9]+$": {
           "$ref": "#/definitions/parameter_set"
         }
       },

beat/core/stats.py

@@ -74,8 +74,6 @@ class Statistics(object):
       errors (list): A list strings containing errors found while loading this
         statistics information.
 
-      data (dict): The original data for these statistics
-
     """
 
     def __init__(self, data=None):

beat/core/test/__init__.py

@@ -35,6 +35,7 @@
 
 # Basic setup for slow tests
 
+import json
 import logging
 import os
 import shutil
@@ -84,14 +85,46 @@ if VERBOSE_TEST_LOGGING:
         logger.addHandler(handler)
 
 
+def sync_prefixes(source_prefixes, target_prefix):
+    for path in source_prefixes:
+        sp.check_call(["rsync", "-arz", path, target_prefix])
+
+
+def initialize_db_root_folder(database_root_folder, databases_path):
+    os.makedirs(database_root_folder, exist_ok=True)
+
+    for root, dirs, files in os.walk(databases_path, topdown=False):
+        for file_ in files:
+            if file_.endswith(".json"):
+                path = os.path.join(root, file_)
+                try:
+                    with open(path, "rt") as db_file:
+                        declaration = json.load(db_file)
+                except json.JSONDecodeError:
+                    # some are explicitly invalid.
+                    continue
+                else:
+                    declaration["root_folder"] = database_root_folder
+                    with open(path, "wt") as db_file:
+                        json.dump(declaration, db_file, indent=4)
+
+
+def setup_root_db_folder():
+    initialize_db_root_folder(
+        os.path.join(prefix_folder, "beat_core_test"), os.path.join(prefix, "databases")
+    )
+
+
 def setup_package():
-    prefixes = [
+    sync_prefixes(
+        [
             pkg_resources.resource_filename("beat.backend.python.test", "prefix"),
             pkg_resources.resource_filename("beat.core.test", "prefix"),
-    ]
+        ],
+        prefix_folder,
+    )
 
-    for path in prefixes:
-        sp.check_call(["rsync", "-arz", path, prefix_folder])
+    setup_root_db_folder()
 
     if DOCKER_NETWORK_TEST_ENABLED:
         import docker

beat/core/test/prefix/algorithms/user-with_dash/block-with_dash/1.json

+{
+    "schema_version": 3,
+    "language": "python",
+    "api_version": 2,
+    "type": "autonomous_loop_processor",
+    "splittable": false,
+    "groups": [
+        {
+            "inputs": {
+                "in": {
+                    "type": "user/single_integer/1"
+                }
+            },
+            "outputs": {
+                "out": {
+                    "type": "user/single_integer/1"
+                }
+            },
+            "loop": {
+                "request": {
+                    "type": "user/1d_array_of_integers/1"
+                },
+                "answer": {
+                    "type": "user/single_float/1"
+                }
+            }
+        }
+    ],
+    "parameters": {
+        "step": {
+            "default": 1,
+            "type": "int8",
+            "description": "Counter step size"
+        }
+    }
+}

beat/core/test/prefix/algorithms/user-with_dash/block-with_dash/1.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+#                                                                                 #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
+# Contact: beat.support@idiap.ch                                                  #
+#                                                                                 #
+# Redistribution and use in source and binary forms, with or without              #
+# modification, are permitted provided that the following conditions are met:     #
+#                                                                                 #
+# 1. Redistributions of source code must retain the above copyright notice, this  #
+# list of conditions and the following disclaimer.                                #
+#                                                                                 #
+# 2. Redistributions in binary form must reproduce the above copyright notice,    #
+# this list of conditions and the following disclaimer in the documentation       #
+# and/or other materials provided with the distribution.                          #
+#                                                                                 #
+# 3. Neither the name of the copyright holder nor the names of its contributors   #
+# may be used to endorse or promote products derived from this software without   #
+# specific prior written permission.                                              #
+#                                                                                 #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
+#                                                                                 #
+###################################################################################
+
+
+import numpy as np
+
+
+class Algorithm:
+    def __init__(self):
+        self.step = 1
+
+    def setup(self, parameters):
+        self.step = parameters["step"]
+        return True
+
+    def process(self, data_loaders, outputs, loop_channel):
+        cnt = 1
+        is_valid, delta = loop_channel.validate({"value": np.full(10, cnt)})
+        while not is_valid:
+            cnt = cnt + self.step
+            is_valid, delta = loop_channel.validate({"value": np.full(10, cnt)})
+
+        delta = delta.value
+
+        data_loader = data_loaders.loaderOf("in")
+
+        for i in range(data_loader.count()):
+            view = data_loader.view("in", i)
+            (data, _, end) = view[view.count() - 1]
+            value = data["in"].value
+            new_value = (value + cnt) * delta
+            outputs["out"].write({"value": np.int32(new_value)}, end)
+
+        return True

beat/core/test/prefix/algorithms/user/integers_rawdata_access/1.json

+{
+    "schema_version": 3,
+    "language": "python",
+    "api_version": 2,
+    "type": "sequential",
+    "splittable": false,
+    "groups": [
+        {
+            "name": "main",
+            "inputs": {
+                "in_data": {
+                    "type": "user/single_integer/1"
+                }
+            },
+            "outputs": {
+                "out_data": {
+                    "type": "user/single_integer/1"
+                }
+            }
+        }
+    ]
+}

beat/core/test/prefix/algorithms/user/integers_rawdata_access/1.py

+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+#                                                                                 #
+# Copyright (c) 2020 Idiap Research Institute, http://www.idiap.ch/               #
+# Contact: beat.support@idiap.ch                                                  #
+#                                                                                 #
+# Redistribution and use in source and binary forms, with or without              #
+# modification, are permitted provided that the following conditions are met:     #
+#                                                                                 #
+# 1. Redistributions of source code must retain the above copyright notice, this  #
+# list of conditions and the following disclaimer.                                #
+#                                                                                 #
+# 2. Redistributions in binary form must reproduce the above copyright notice,    #
+# this list of conditions and the following disclaimer in the documentation       #
+# and/or other materials provided with the distribution.                          #
+#                                                                                 #
+# 3. Neither the name of the copyright holder nor the names of its contributors   #
+# may be used to endorse or promote products derived from this software without   #
+# specific prior written permission.                                              #
+#                                                                                 #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
+#                                                                                 #
+###################################################################################
+
+
+import numpy
+
+
+class Algorithm:
+    def process(self, inputs, data_loaders, outputs):
+        with open(
+            "/databases/simple_rawdata_access/1/datafile.txt", "rt"
+        ) as shared_data:
+            value = shared_data.read()
+            shared_offset = int(value)
+
+        out_data = {"value": numpy.int32(inputs["in_data"].data.value + shared_offset)}
+        outputs["out_data"].write(out_data)
+
+        return True

beat/core/test/prefix/databases/empty_protocol_sets/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/empty_protocols/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
     ]
 }

beat/core/test/prefix/databases/empty_set_outputs/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/integers_db_invalid_env/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "environment": {
         "name": "Does not exist",
         "version": "1.4.0"

beat/core/test/prefix/databases/integers_db_invalid_env/2.json

@@ -4,7 +4,7 @@
         "name": "Does not exist",
         "version": "1.4.0"
     },
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/large/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "large",

beat/core/test/prefix/databases/large/2.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "large",

beat/core/test/prefix/databases/missing_protocol_name/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "sets": [

beat/core/test/prefix/databases/missing_protocols/1.json

 {
-    "root_folder": "/tmp/path/not/set"
+    "root_folder": "/tmp/beat_core_test"
 }

beat/core/test/prefix/databases/missing_set_name/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/missing_set_outputs/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/missing_set_view/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/mixed_protocol_names/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "protocol",

beat/core/test/prefix/databases/mixed_set_names/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/same_protocol_names/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/same_set_names/1.json

 {
-    "root_folder": "/tmp/path/not/set",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "double",

beat/core/test/prefix/databases/simple/1.json

 {
-  "root_folder": "/tmp/foo/bar",
+  "root_folder": "/tmp/beat_core_test",
   "protocols": [
     {
       "name": "protocol",

beat/core/test/prefix/databases/simple/2.json

 {
-    "root_folder": "/tmp/foo/bar",
+    "root_folder": "/tmp/beat_core_test",
     "protocols": [
         {
             "name": "protocol",

beat/core/test/prefix/databases/simple_env/1.json

 {
-  "root_folder": "/tmp/foo/bar",
+  "root_folder": "/tmp/beat_core_test",
   "environment": {
       "name": "Example databases",
       "version": "1.4.0"

beat/core/test/prefix/databases/simple_env/2.json

 {
-    "root_folder": "/tmp/foo/bar",
+    "root_folder": "/tmp/beat_core_test",
     "environment": {
       "name": "Example databases",
       "version": "1.4.0"

beat/core/test/prefix/databases/simple_invalid_env/1.json

 {
-  "root_folder": "/tmp/foo/bar",
+  "root_folder": "/tmp/beat_core_test",
   "environment": {
       "name": "Not existing",
       "version": "1.4.0"

beat/core/test/prefix/databases/simple_invalid_env/2.json

 {
-    "root_folder": "/tmp/foo/bar",
+    "root_folder": "/tmp/beat_core_test",
     "environment": {
       "name": "Not existing",
       "version": "1.4.0"

beat/core/test/prefix/databases/simple_rawdata_access/1.json

+{
+  "root_folder": "/tmp/beat_core_test",
+  "environment": {
+      "name": "Example databases",
+      "version": "1.4.0"
+    },
+  "direct_rawdata_access": true,
+  "protocols": [
+    {
+      "name": "protocol",
+      "template": "test_integers",
+      "sets": [
+        {
+          "name": "set",
+          "template": "set",
+          "view": "View",
+          "outputs": {
+            "out": "user/single_integer/1"
+          }
+        },
+        {
+          "name": "set2",
+          "template": "set",
+          "view": "View2",
+          "outputs": {
+            "out": "user/single_integer/1"
+          }
+        }
+      ]
+    },
+    {
+      "name": "protocol2",
+      "template": "test_integers",
+      "sets": [
+        {
+          "name": "set",
+          "template": "set",
+          "view": "LargeView",
+          "outputs": {
+            "out": "user/single_integer/1"
+          }
+        },
+        {
+          "name": "set2",
+          "template": "set",
+          "view": "View2",
+          "outputs": {
+            "out": "user/single_integer/1"
+          }
+        }
+      ]
+    }
+  ]
+}

beat/core/test/prefix/databases/simple_rawdata_access/1.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+#                                                                                 #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
+# Contact: beat.support@idiap.ch                                                  #
+#                                                                                 #
+# Redistribution and use in source and binary forms, with or without              #
+# modification, are permitted provided that the following conditions are met:     #
+#                                                                                 #
+# 1. Redistributions of source code must retain the above copyright notice, this  #
+# list of conditions and the following disclaimer.                                #
+#                                                                                 #
+# 2. Redistributions in binary form must reproduce the above copyright notice,    #
+# this list of conditions and the following disclaimer in the documentation       #
+# and/or other materials provided with the distribution.                          #
+#                                                                                 #
+# 3. Neither the name of the copyright holder nor the names of its contributors   #
+# may be used to endorse or promote products derived from this software without   #
+# specific prior written permission.                                              #
+#                                                                                 #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
+#                                                                                 #
+###################################################################################
+
+
+from collections import namedtuple
+
+import numpy
+
+from beat.backend.python.database import View as BaseView
+
+
+class View(BaseView):
+    def setup(
+        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+    ):
+        """Initializes the database"""
+
+        super().setup(root_folder, outputs, parameters, start_index, end_index)
+
+        return True
+
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(42)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}
+
+
+# ----------------------------------------------------------
+
+
+class View2(BaseView):
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(53)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}
+
+
+# ----------------------------------------------------------
+
+
+class LargeView(BaseView):
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(0), Entry(1), Entry(2), Entry(3), Entry(4)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}

beat/core/test/prefix/databases/simple_rawdata_access/1.rst

+
+.. Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.backend.python module of the BEAT platform.  ..
+..                                                                            ..
+.. Redistribution and use in source and binary forms, with or without
+.. modification, are permitted provided that the following conditions are met:
+
+.. 1. Redistributions of source code must retain the above copyright notice, this
+.. list of conditions and the following disclaimer.
+
+.. 2. Redistributions in binary form must reproduce the above copyright notice,
+.. this list of conditions and the following disclaimer in the documentation
+.. and/or other materials provided with the distribution.
+
+.. 3. Neither the name of the copyright holder nor the names of its contributors
+.. may be used to endorse or promote products derived from this software without
+.. specific prior written permission.
+
+.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+.. ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+.. WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.. DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+.. FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+.. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+.. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+The Simple Digit Database
+-------------------------
+
+This database emits two integers (one from each of its set). The first integer
+(from ``set``) has a value of 42, while the second (from ``set2``), a value of
+53.

beat/core/test/prefix/databases/simple_rawdata_access/2.json

+{
+    "root_folder": "/tmp/beat_core_test",
+    "environment": {
+      "name": "Example databases",
+      "version": "1.4.0"
+    },
+    "direct_rawdata_access": true,
+    "protocols": [
+        {
+            "name": "protocol",
+            "template": "protocol/1",
+            "views": {
+                "set": {
+                    "view": "View"
+                },
+                "set2": {
+                    "view": "View2"
+                }
+            }
+        },
+        {
+            "name": "protocol2",
+            "template": "protocol2/1",
+            "views": {
+                "set": {
+                    "view": "LargeView"
+                },
+                "set2": {
+                    "view": "View2"
+                }
+            }
+        }
+    ],
+    "schema_version": 2
+}

beat/core/test/prefix/databases/simple_rawdata_access/2.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+#                                                                                 #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
+# Contact: beat.support@idiap.ch                                                  #
+#                                                                                 #
+# Redistribution and use in source and binary forms, with or without              #
+# modification, are permitted provided that the following conditions are met:     #
+#                                                                                 #
+# 1. Redistributions of source code must retain the above copyright notice, this  #
+# list of conditions and the following disclaimer.                                #
+#                                                                                 #
+# 2. Redistributions in binary form must reproduce the above copyright notice,    #
+# this list of conditions and the following disclaimer in the documentation       #
+# and/or other materials provided with the distribution.                          #
+#                                                                                 #
+# 3. Neither the name of the copyright holder nor the names of its contributors   #
+# may be used to endorse or promote products derived from this software without   #
+# specific prior written permission.                                              #
+#                                                                                 #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
+#                                                                                 #
+###################################################################################
+
+
+from collections import namedtuple
+
+import numpy
+
+from beat.backend.python.database import View as BaseView
+
+
+class View(BaseView):
+    def setup(
+        self, root_folder, outputs, parameters, start_index=None, end_index=None,
+    ):
+        """Initializes the database"""
+
+        super().setup(root_folder, outputs, parameters, start_index, end_index)
+
+        return True
+
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(42)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}
+
+
+# ----------------------------------------------------------
+
+
+class View2(BaseView):
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(53)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}
+
+
+# ----------------------------------------------------------
+
+
+class LargeView(BaseView):
+    def index(self, root_folder, parameters):
+        Entry = namedtuple("Entry", ["out"])
+
+        return [Entry(0), Entry(1), Entry(2), Entry(3), Entry(4)]
+
+    def get(self, output, index):
+        obj = self.objs[index]
+
+        if output == "out":
+            return {"value": numpy.int32(obj.out)}

beat/core/test/prefix/experiments/errors/user/single/1/single_no_rawdata_access.json

+{
+  "analyzers": {
+    "analysis": {
+      "algorithm": "v1/integers_echo_analyzer/1",
+      "inputs": {
+        "in_data": "in"
+      }
+    }
+  },
+  "blocks": {
+    "echo": {
+      "algorithm": "user/integers_rawdata_access/1",
+      "inputs": {
+        "in_data": "in"
+      },
+      "outputs": {
+        "out_data": "out"
+      }
+    }
+  },
+  "datasets": {
+    "set": {
+      "database": "simple/1",
+      "protocol": "protocol",
+      "set": "set"
+    }
+  },
+  "globals": {
+    "queue": "queue",
+    "environment": {
+      "name": "Python for tests",
+      "version": "1.3.0"
+    }
+  }
+}

beat/core/test/prefix/experiments/errors/user/single/1/single_no_rawdata_access.rst

+Test experiment for validating the rawdata access feature. This ensure that if the
+database does not have the field then nothing is mounted.

beat/core/test/prefix/experiments/user/user/loop/1/with_dash.json

+{
+    "schema_version": 2,
+    "blocks": {
+    },
+    "loops": {
+        "loop_super_block": {
+            "processor_algorithm": "user-with_dash/block-with_dash/1",
+            "processor_parameters": {
+                "step": 1
+            },
+            "processor_inputs": {
+                "in": "in"
+            },
+            "processor_outputs": {
+                "out": "out"
+            },
+
+            "evaluator_algorithm": "user/db_input_loop_evaluator/1",
+            "evaluator_parameters": {
+                "threshold": 9
+            },
+            "evaluator_inputs": {
+                "in_loop": "in_loop"
+            },
+            "evaluator_outputs": {
+                "out_loop": "out_loop"
+            }
+        }
+    },
+    "analyzers": {
+        "analysis": {
+            "algorithm": "v1/integers_analysis/1",
+            "parameters": {
+            },
+            "inputs": {
+                "input": "input"
+            }
+        },
+        "loop_analysis": {
+            "algorithm": "v1/integers_analysis/1",
+            "parameters": {
+            },
+            "inputs": {
+                "input": "input"
+            }
+        }
+    },
+    "datasets": {
+        "integers": {
+            "database": "integers_db/1",
+            "protocol": "double",
+            "set": "double"
+        },
+        "integers2": {
+            "database": "simple/1",
+            "protocol": "protocol2",
+            "set": "set"
+        }
+    },
+    "globals": {
+        "environment": {
+            "name": "Python for tests",
+            "version": "1.3.0"
+        },
+        "queue": "queue",
+        "user-with_dash/block-with_dash/1": {
+            "step": 2
+        },
+        "user/db_input_loop_evaluator/1" : {
+            "threshold": 8
+        }
+    }
+}

beat/core/test/prefix/experiments/user/user/loop/1/with_dash.rst

+Test that loading an experiment with an algorithm containg dashes
+in its author name works.
+
+See beat/beat.core#104

beat/core/test/prefix/experiments/user/user/single/1/single_rawdata_access.json

+{
+  "analyzers": {
+    "analysis": {
+      "algorithm": "v1/integers_echo_analyzer/1",
+      "inputs": {
+        "in_data": "in"
+      }
+    }
+  },
+  "blocks": {
+    "echo": {
+      "algorithm": "user/integers_rawdata_access/1",
+      "inputs": {
+        "in_data": "in"
+      },
+      "outputs": {
+        "out_data": "out"
+      }
+    }
+  },
+  "datasets": {
+    "set": {
+      "database": "simple_rawdata_access/1",
+      "protocol": "protocol",
+      "set": "set"
+    }
+  },
+  "globals": {
+    "queue": "queue",
+    "environment": {
+      "name": "Python for tests",
+      "version": "1.3.0"
+    }
+  }
+}

beat/core/test/prefix/experiments/user/user/single/1/single_rawdata_access.rst

+Test experiment for validating the rawdata access feature

beat/core/test/test_database.py

@@ -47,6 +47,7 @@ def test_export():
         yield export, f"integers_db/{i}"
         yield export, f"simple/{i}"
         yield export, f"large/{i}"
+        yield export, f"simple_rawdata_access/{i}"
 
 
 @nose.tools.with_setup(teardown=cleanup)
@@ -60,3 +61,22 @@ def export(db_name):
     # load from tmp_prefix and validates
     exported = Database(tmp_prefix, db_name)
     nose.tools.assert_true(exported.valid, "\n  * %s" % "\n  * ".join(exported.errors))
+
+
+def test_rawdata_access():
+    for i in range(1, 3):
+        yield rawdata_access, f"integers_db/{i}", False
+        yield rawdata_access, f"simple/{i}", False
+        yield rawdata_access, f"large/{i}", False
+        yield rawdata_access, f"simple_rawdata_access/{i}", True
+
+
+@nose.tools.with_setup(teardown=cleanup)
+def rawdata_access(db_name, rawdata_access_enabled):
+
+    obj = Database(prefix, db_name)
+
+    nose.tools.assert_true(obj.valid, "\n  * %s" % "\n  * ".join(obj.errors))
+    nose.tools.assert_equal(
+        obj.is_database_rawdata_access_enabled, rawdata_access_enabled
+    )

beat/core/test/test_docker.py

@@ -107,8 +107,7 @@ class NetworkTest(NoDiscoveryTests):
 class UserTest(NoDiscoveryTests):
     @slow
     def test_user(self):
-        """Test that the uid property is correctly used.
-        """
+        """Test that the uid property is correctly used."""
 
         container = self.host.create_container("debian:8.4", ["id"])
         container.uid = 10000
@@ -127,8 +126,7 @@ class UserTest(NoDiscoveryTests):
 class EnvironmentVariableTest(NoDiscoveryTests):
     @slow
     def test_environment_variable(self):
-        """Test that the uid property is correctly used.
-        """
+        """Test that the uid property is correctly used."""
 
         container = self.host.create_container("debian:8.4", ["env"])
         container.add_environment_variable("DOCKER_TEST", "good")
@@ -143,8 +141,7 @@ class EnvironmentVariableTest(NoDiscoveryTests):
 class WorkdirTest(NoDiscoveryTests):
     @slow
     def test_workdir(self):
-        """Test that the workdir property is correctly used.
-        """
+        """Test that the workdir property is correctly used."""
 
         with TemporaryDirectory() as tmp_folder:
             test_file = "test.txt"
@@ -169,8 +166,7 @@ class WorkdirTest(NoDiscoveryTests):
 class EntrypointTest(NoDiscoveryTests):
     @slow
     def test_entrypoint(self):
-        """Test that the entrypoint property is correctly used.
-        """
+        """Test that the entrypoint property is correctly used."""
 
         container = self.host.create_container("debian:8.4", ["42"])
         container.set_entrypoint("echo")
@@ -187,8 +183,7 @@ class EntrypointTest(NoDiscoveryTests):
 
 class TmpfsTest(NoDiscoveryTests):
     def test_tmpfs(self):
-        """Test that the tmpfs are properly mounted and usable.
-        """
+        """Test that the tmpfs are properly mounted and usable."""
 
         container = self.host.create_container(
             "debian:8.4", ["touch", "/dummy/test.txt"]
@@ -213,33 +208,6 @@ class TmpfsTest(NoDiscoveryTests):
         self.assertEqual(status, 0)
         self.assertEqual(logs, "")
 
-    def test_tmpfs_size(self):
-        """Test that the tmpfs are respected.
-        """
-
-        container = self.host.create_container(
-            "debian:8.4", ["dd", "if=/dev/zero", "of=/dummy/test.txt"]
-        )
-
-        tmpfs_list = container.temporary_filesystems
-
-        self.assertEqual(len(tmpfs_list), 2)
-
-        container.add_tmpfs("/dummy", "1M")
-
-        tmpfs_list = container.temporary_filesystems
-
-        self.assertEqual(len(tmpfs_list), 3)
-
-        self.host.start(container)
-        status = self.host.wait(container)
-        logs = self.host.logs(container)
-        if status != 0:
-            print(logs)
-
-        self.assertEqual(status, 1)
-        self.assertTrue("No space left" in logs)
-
 
 class AsyncTest(NoDiscoveryTests):
     @slow
@@ -300,7 +268,7 @@ class AsyncTest(NoDiscoveryTests):
         self.assertEqual(self.host.logs(container), "")
 
 
-class AsyncWithEnvironmentTest(unittest.TestCase):
+class WithDiscoveryTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.host = Host(raise_on_errors=False)
@@ -314,6 +282,31 @@ class AsyncWithEnvironmentTest(unittest.TestCase):
         self.host.teardown()
         self.assertFalse(self.host.containers)  # All containers are gone
 
+
+class TmpfsWithEnvironmentTest(WithDiscoveryTests):
+    def test_tmpfs_from_label(self):
+        """Test that the tmpfs are respected."""
+
+        container = self.host.create_container(
+            "Python for tests (1.3.0)",
+            ["dd", "if=/dev/zero", "of=/custom_tmpfs/test.txt"],
+        )
+
+        tmpfs_list = container.temporary_filesystems
+
+        self.assertEqual(len(tmpfs_list), 3)
+        print(tmpfs_list)
+        self.host.start(container)
+        status = self.host.wait(container)
+        logs = self.host.logs(container)
+        if status != 0:
+            print(logs)
+
+        self.assertEqual(status, 1)
+        self.assertTrue("No space left" in logs)
+
+
+class AsyncWithEnvironmentTest(WithDiscoveryTests):
     @slow
     def test_memory_limit(self):
 

beat/core/test/test_docker_execution.py

@@ -41,11 +41,15 @@ import subprocess  # nosec
 
 import nose.tools
 
+from beat.core.database import Database
+
 from ..dock import Host
 from ..execution import DockerExecutor
 from . import DOCKER_NETWORK_TEST_ENABLED
 from . import network_name
+from . import prefix as test_prefix
 from . import prefix_folder
+from . import setup_root_db_folder
 from .test_execution import BaseExecutionMixIn
 from .utils import DOCKER_TEST_IMAGES
 from .utils import cleanup
@@ -60,6 +64,21 @@ BUILDER_IMAGE = (
 # ----------------------------------------------------------
 
 
+def write_rawdata_for_database(database_name, raw_data):
+    """Generate raw data for give database"""
+
+    db = Database(test_prefix, database_name)
+    nose.tools.assert_true(db.valid, db.errors)
+
+    data_sharing_path = db.data["root_folder"]
+
+    with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
+        data_file.write("{}".format(raw_data))
+
+
+# ----------------------------------------------------------
+
+
 class TestDockerExecution(BaseExecutionMixIn):
     @classmethod
     def setup_class(cls):
@@ -127,6 +146,8 @@ class TestDockerExecution(BaseExecutionMixIn):
             ]
         )
 
+        setup_root_db_folder()
+
     @slow
     @skipif(not DOCKER_NETWORK_TEST_ENABLED, "Network test disabled")
     def test_custom_network(self):
@@ -148,6 +169,27 @@ class TestDockerExecution(BaseExecutionMixIn):
 
         nose.tools.assert_is_none(result)
 
+    @slow
+    def test_database_rawdata_access(self):
+        offset = 12
+
+        write_rawdata_for_database("simple_rawdata_access/1", offset)
+
+        result = self.execute(
+            "user/user/single/1/single_rawdata_access", [{"out_data": 42 + offset}]
+        )
+
+        nose.tools.assert_is_none(result)
+
+    @slow
+    def test_database_no_rawdata_access(self):
+        write_rawdata_for_database("simple/1", "should not be loaded")
+
+        result = self.execute("errors/user/single/1/single_no_rawdata_access", [None])
+
+        nose.tools.eq_(result["status"], 1)
+        nose.tools.assert_true("FileNotFoundError" in result["user_error"])
+
     @slow
     def test_single_1_prepare_error(self):