diff --git a/beat/core/database.py b/beat/core/database.py
index 2c200742a20d88cd0b5a8173cc40e77ffc9ef6d0..ce60f72eef728cec92e0286a318254655a3d03a8 100644
--- a/beat/core/database.py
+++ b/beat/core/database.py
@@ -102,6 +102,15 @@ class Database(BackendDatabase):
def __init__(self, prefix, data, dataformat_cache=None):
super(Database, self).__init__(prefix, data, dataformat_cache)
+ def _validate_view(self, view_name):
+ if view_name.find(".") != -1 or view_name.find(os.sep) != -1:
+ self.errors.append(
+ "dataset views are required to sit inside the "
+ "database root folder, but `%s' is either in a "
+ "subdirectory or points to a python module, what is "
+ "unsupported by this version" % (view_name)
+ )
+
def _load(self, data, dataformat_cache):
"""Loads the database"""
@@ -150,7 +159,7 @@ class Database(BackendDatabase):
self._validate_semantics(dataformat_cache)
def _validate_semantics(self, dataformat_cache):
- """Validates all sematical aspects of the database"""
+ """Validates all semantical aspects of the database"""
# all protocol names must be unique
protocol_names = [k["name"] for k in self.data["protocols"]]
@@ -161,7 +170,7 @@ class Database(BackendDatabase):
# all set names within a protocol must be unique
for protocol in self.data["protocols"]:
- set_names = [k["name"] for k in protocol["sets"]]
+ set_names = self.set_names(protocol["name"])
if len(set_names) != len(set(set_names)):
self.errors.append(
"found different sets with the same name at protocol "
@@ -169,9 +178,9 @@ class Database(BackendDatabase):
)
# all outputs must have valid data types
- for _set in protocol["sets"]:
+ for _, set_ in self.sets(protocol["name"]).items():
- for key, value in _set["outputs"].items():
+ for key, value in set_["outputs"].items():
if value in self.dataformats:
continue
@@ -191,17 +200,16 @@ class Database(BackendDatabase):
% (
value,
key,
- _set["name"],
+ set_["name"],
protocol["name"],
"\n".join(dataformat.errors),
)
)
# all view names must be relative to the database root path
- if _set["view"].find(".") != -1 or _set["view"].find(os.sep) != -1:
- self.errors.append(
- "dataset views are required to sit inside the "
- "database root folder, but `%s' is either in a "
- "subdirectory or points to a python module, what is "
- "unsupported by this version" % (_set["view"],)
- )
+ if self.schema_version == 1:
+ self._validate_view(set_["view"])
+
+ if self.schema_version != 1:
+ for view in protocol["views"].keys():
+ self._validate_view(view)
diff --git a/beat/core/protocoltemplate.py b/beat/core/protocoltemplate.py
new file mode 100644
index 0000000000000000000000000000000000000000..11574f56ce556c944fd57238b466cd84c4c80aa4
--- /dev/null
+++ b/beat/core/protocoltemplate.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+# #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are met: #
+# #
+# 1. Redistributions of source code must retain the above copyright notice, this #
+# list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright notice, #
+# this list of conditions and the following disclaimer in the documentation #
+# and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its contributors #
+# may be used to endorse or promote products derived from this software without #
+# specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+###################################################################################
+
+
+"""
+================
+protocoltemplate
+================
+
+Validation of protocoltemplate
+
+Forward importing from :py:mod:`beat.backend.python.protocoltemplate`:
+:py:class:`beat.backend.python.protocoltemplate.Storage`
+"""
+
+import six
+
+from . import schema
+
+from beat.backend.python.protocoltemplate import Storage
+from beat.backend.python.protocoltemplate import (
+ ProtocolTemplate as BackendProtocolTemplate,
+)
+
+
+class ProtocolTemplate(BackendProtocolTemplate):
+ """Protocol template define the design of the database.
+
+
+ Parameters:
+
+ prefix (str): Establishes the prefix of your installation.
+
+ data (dict, str): The piece of data representing the protocol templates.
+ It must validate against the schema defined for protocol templates. If a
+ string is passed, it is supposed to be a valid path to protocol template
+ in the designated prefix area.
+
+ dataformat_cache (:py:class:`dict`, Optional): A dictionary mapping
+ dataformat names to loaded dataformats. This parameter is optional and,
+ if passed, may greatly speed-up protocol template loading times as
+ dataformats that are already loaded may be re-used. If you use this
+ parameter, you must guarantee that the cache is refreshed as appropriate
+ in case the underlying dataformats change.
+
+
+ Attributes:
+
+ name (str): The full, valid name of this protocol template
+
+ description (str): The short description string, loaded from the JSON
+ file if one was set.
+
+ documentation (str): The full-length docstring for this object.
+
+ storage (object): A simple object that provides information about file
+ paths for this protocol template
+
+ errors (list): A list containing errors found while loading this
+ protocol template.
+
+ data (dict): The original data for this protocol template, as loaded by
+ our JSON decoder.
+
+ """
+
+ def __init__(self, prefix, data, dataformat_cache=None):
+ super(ProtocolTemplate, self).__init__(prefix, data, dataformat_cache)
+
+ def _load(self, data, dataformat_cache):
+ """Loads the database"""
+
+ self._name = None
+ self.storage = None
+ self.dataformats = {} # preloaded dataformats
+
+ if isinstance(data, six.string_types): # user has passed a file pointer
+
+ self._name = data
+ self.storage = Storage(self.prefix, self._name)
+ data = self.storage.json.path
+ if not self.storage.json.exists():
+ self.errors.append(
+ "Protocol template declaration file not found: %s" % data
+ )
+ return
+
+ # this runs basic validation, including JSON loading if required
+ self.data, self.errors = schema.validate("protocoltemplate", data)
+ if self.errors:
+ return # don't proceed with the rest of validation
diff --git a/beat/core/schema/__init__.py b/beat/core/schema/__init__.py
index ae809022eb9a8b3ee7f337259770e44b9794cbdd..d182bf356edc2af58ba47bb9edd5918ceff0bbdf 100644
--- a/beat/core/schema/__init__.py
+++ b/beat/core/schema/__init__.py
@@ -94,7 +94,11 @@ def load_schema(schema_name, version=1):
with open(fname, "rb") as f:
data = f.read().decode()
- schema = json.loads(data)
+ try:
+ schema = json.loads(data)
+ except json.errors.JSONDecodeError:
+ print("Invalid json:\n {data}".format(data))
+ raise
basedir = os.path.realpath(os.path.dirname(fname))
resolver = jsonschema.RefResolver("file://" + basedir + "/", schema)
diff --git a/beat/core/schema/database/2.json b/beat/core/schema/database/2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea20d45ec57865ceb0be70d9477a2c57cc0b9b2f
--- /dev/null
+++ b/beat/core/schema/database/2.json
@@ -0,0 +1,107 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Database descriptor v2",
+ "description": "This schema defines the properties of a version 2 BEAT database",
+
+ "type": "object",
+
+ "properties": {
+
+ "root_folder": {
+ "type": "string",
+ "pattern": "^((file://)?(/[^/]+)+|nfs://[a-z0-9._-]+:(/[^/]+)+)$"
+ },
+
+ "protocols": {
+ "type": "array",
+ "minItems": 1,
+ "uniqueItems": true,
+ "items": { "$ref": "#/definitions/protocol" }
+ },
+
+ "description": { "$ref": "../common/1.json#/definitions/description" },
+
+ "schema_version": { "const": 2 }
+
+ },
+
+ "required": [
+ "root_folder",
+ "protocols",
+ "schema_version"
+ ],
+
+ "additionalProperties": false,
+
+ "definitions": {
+
+ "template_identifier": {
+ "type": "string",
+ "pattern": "^[a-zA-Z0-9_-]+/[0-9]+$"
+ },
+
+ "protocol": {
+ "type": "object",
+ "properties": {
+ "name": { "$ref": "#/definitions/protocol_name" },
+ "template": { "$ref": "#/definitions/template_identifier" },
+ "views": { "$ref": "#/definitions/views" }
+ },
+ "required": ["name", "views", "template"],
+ "additionalProperties": false
+ },
+
+ "protocol_name": {
+ "type": "string",
+ "pattern": "^[a-zA-Z0-9_][\\.a-zA-Z0-9_-]*$"
+ },
+
+ "view_name": {
+ "type": "string",
+ "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"
+ },
+
+ "views": {
+ "type": "object",
+ "minProperties": 1,
+ "uniqueItems": true,
+ "patternProperties": {
+ "^[a-zA-Z_][a-zA-Z0-9_]*$": { "$ref": "#/definitions/view" }
+ }
+ },
+
+ "view": {
+ "type": "object",
+ "properties": {
+ "view": { "$ref": "#definitions/view_name" },
+ "parameters": { "$ref": "#/definitions/parameters" }
+ },
+ "additionalProperties": false
+ },
+
+ "parameters": {
+ "type": "object",
+ "patternProperties": {
+ "^[a-zA-Z_][a-zA-Z0-9_-]*$": {
+ "$ref": "#/definitions/parameter_value"
+ }
+ }
+ },
+
+ "parameter_value": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "number"
+ },
+ {
+ "type": "boolean"
+ }
+ ]
+ }
+
+ }
+
+}
diff --git a/beat/core/schema/protocoltemplate/1.json b/beat/core/schema/protocoltemplate/1.json
new file mode 100644
index 0000000000000000000000000000000000000000..07af41a741ea69f04136dfa669179bdfe0788171
--- /dev/null
+++ b/beat/core/schema/protocoltemplate/1.json
@@ -0,0 +1,58 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Database Protocol descriptor",
+ "description": "This schema defines the properties of a BEAT database protocol",
+
+ "type": "object",
+
+ "properties": {
+
+ "description": { "$ref": "../common/1.json#/definitions/description" },
+
+ "schema_version": { "const": 1 },
+
+ "sets": {
+ "type": "array",
+ "minItems": 1,
+ "uniqueItems": true,
+ "items": { "$ref": "#/definitions/set" }
+ }
+
+ },
+
+ "required": [
+ "schema_version", "sets"
+ ],
+
+ "additionalProperties": false,
+
+ "definitions": {
+
+ "identifier": {
+ "type": "string",
+ "pattern": "^[a-zA-Z_][a-zA-Z0-9_-]*$"
+ },
+
+ "set": {
+ "type": "object",
+ "properties": {
+ "name": { "$ref": "#/definitions/identifier" },
+ "outputs": {
+ "type": "object",
+ "patternProperties": {
+ "^[a-zA-Z_][a-zA-Z0-9_-]*$": {
+ "$ref": "../common/1.json#/definitions/reference"
+ }
+ },
+ "minProperties": 1,
+ "uniqueItems": true,
+ "additionalProperties": false
+ }
+ },
+ "required": ["name", "outputs"],
+ "additionalProperties": false
+ }
+
+ }
+
+}
diff --git a/beat/core/scripts/migrate_db_to_v2.py b/beat/core/scripts/migrate_db_to_v2.py
new file mode 100644
index 0000000000000000000000000000000000000000..707eb5d910bf6cb31da99d9bb1e960290ae775b0
--- /dev/null
+++ b/beat/core/scripts/migrate_db_to_v2.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+# #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are met: #
+# #
+# 1. Redistributions of source code must retain the above copyright notice, this #
+# list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright notice, #
+# this list of conditions and the following disclaimer in the documentation #
+# and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its contributors #
+# may be used to endorse or promote products derived from this software without #
+# specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+###################################################################################
+
+"""Migrate a v1 database to v2
+
+Usage:
+ %(prog)s [-v ... | --verbose ...] [--prefix=<path>][-f|--force]
+ <database_identifier>
+ %(prog)s (--help | -h)
+ %(prog)s (--version | -V)
+
+
+Options:
+ -h, --help Show this screen
+ -V, --version Show version
+ -v, --verbose Increases the output verbosity level
+ -p, --prefix=<path> Path where the prefix is contained [default: .]
+"""
+
+import os
+import sys
+import copy
+
+from docopt import docopt
+
+from ..version import __version__
+
+from ..database import Database, Storage as DBStorage
+from ..protocoltemplate import ProtocolTemplate, Storage as PTStorage
+from ..utils import setup_logging
+
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+
+ prog = os.path.basename(sys.argv[0])
+ completions = dict(prog=prog, version=__version__)
+ args = docopt(
+ __doc__ % completions,
+ argv=argv,
+ options_first=True,
+ version="v%s" % __version__,
+ )
+
+ logger = setup_logging(args["--verbose"], __name__, __name__)
+
+ prefix = args["--prefix"] if args["--prefix"] is not None else "."
+ if not os.path.exists(prefix):
+ logger.error("Prefix not found at: '%s'", prefix)
+ return 1
+
+ database_identifier = args["<database_identifier>"]
+
+ database = Database(prefix, database_identifier)
+
+ if not database.valid:
+ logger.error("Invalid database: '%s'", "\n".join(database.errors))
+ return 1
+
+ if database.schema_version != 1:
+ logger.error("Can't migrate database is not v1")
+ return 1
+
+ db_name, db_version = database_identifier.split("/")
+ new_db_name = f"{db_name}/{int(db_version) + 1}"
+
+ db_storage = DBStorage(prefix, new_db_name)
+ if db_storage.exists():
+ logger.error(f"Database already exists: {new_db_name}")
+ return 1
+
+ database_json = copy.deepcopy(database.data)
+ database_json["schema_version"] = 2
+ database_json["protocols"] = []
+
+ for protocol in database.protocols:
+ sets = database.sets(protocol)
+ set_list = []
+ views = {}
+ for _, set_ in sets.items():
+ views[set_["name"]] = {
+ "view": set_["view"],
+ "parameters": set_.get("parameters", {}),
+ }
+
+ for key in ["template", "view", "parameters"]:
+ if key in set_:
+ set_.pop(key)
+ set_list.append(set_)
+
+ template = {"schema_version": 1, "sets": set_list}
+
+ pt_name = f"{protocol}/1"
+ pt_storage = PTStorage(prefix, pt_name)
+
+ if pt_storage.exists():
+ logger.info(f"Protocol template already exists: {pt_name}")
+ else:
+ protocol_template = ProtocolTemplate(prefix, template)
+ if not protocol_template.valid:
+ logger.error(
+ "Invalid protocol created:", "\n".join(protocol_template.errors)
+ )
+ return 1
+ else:
+ protocol_template.write(pt_storage)
+
+ protocol_entry = {"name": protocol, "template": pt_name, "views": views}
+
+ database_json["protocols"].append(protocol_entry)
+
+ new_database = Database(prefix, database_json)
+ if not new_database.valid:
+ logger.error("Invalid database created:", "\n".join(new_database.errors))
+ return 1
+ else:
+ new_database.code = database.code
+ new_database.description = (
+ database.description if database.description is not None else ""
+ )
+ new_database.write(db_storage)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/beat/core/test/prefix/databases/large/2.json b/beat/core/test/prefix/databases/large/2.json
new file mode 100644
index 0000000000000000000000000000000000000000..629eab25e7f362e76b03cb4c87c0c1a7131adb4d
--- /dev/null
+++ b/beat/core/test/prefix/databases/large/2.json
@@ -0,0 +1,27 @@
+{
+ "root_folder": "/tmp/path/not/set",
+ "protocols": [
+ {
+ "name": "large",
+ "template": "large/1",
+ "views": {
+ "data": {
+ "view": "LargeView",
+ "parameters": {}
+ }
+ }
+ },
+ {
+ "name": "small",
+ "template": "small/1",
+ "views": {
+ "data": {
+ "view": "SmallView",
+ "parameters": {}
+ }
+ }
+ }
+ ],
+ "schema_version": 2,
+ "description": ""
+}
diff --git a/beat/core/test/prefix/databases/large/2.py b/beat/core/test/prefix/databases/large/2.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a5a976c7b6d8e921b0b4653052cd375d35a4d18
--- /dev/null
+++ b/beat/core/test/prefix/databases/large/2.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+# #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are met: #
+# #
+# 1. Redistributions of source code must retain the above copyright notice, this #
+# list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright notice, #
+# this list of conditions and the following disclaimer in the documentation #
+# and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its contributors #
+# may be used to endorse or promote products derived from this software without #
+# specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+###################################################################################
+
+
+import numpy
+from collections import namedtuple
+from beat.backend.python.database import View
+
+
+# ----------------------------------------------------------
+
+
+class LargeView(View):
+ def __init__(self):
+ super(LargeView, self).__init__()
+ numpy.random.seed(0) # So it is kept reproducible
+
+ def index(self, root_folder, parameters):
+ Entry = namedtuple("Entry", ["out"])
+
+ entries = []
+ for i in range(0, 1000):
+ entries.append(Entry(numpy.int32(numpy.random.randint(100, size=(1000,)))))
+
+ return entries
+
+ def get(self, output, index):
+ obj = self.objs[index]
+
+ if output == "out":
+ return {"value": obj.out}
+
+
+# ----------------------------------------------------------
+
+
+class SmallView(View):
+ def __init__(self):
+ super(SmallView, self).__init__()
+ numpy.random.seed(0) # So it is kept reproducible
+
+ def index(self, root_folder, parameters):
+ Entry = namedtuple("Entry", ["out"])
+
+ entries = []
+ for i in range(0, 1000):
+ entries.append(Entry(numpy.int32(numpy.random.randint(0, 100))))
+
+ return entries
+
+ def get(self, output, index):
+ obj = self.objs[index]
+
+ if output == "out":
+ return {"value": obj.out}
diff --git a/beat/core/test/prefix/databases/simple/2.json b/beat/core/test/prefix/databases/simple/2.json
new file mode 100644
index 0000000000000000000000000000000000000000..f871a13fa326439abd5dc31699b4fa3d3ac2ad34
--- /dev/null
+++ b/beat/core/test/prefix/databases/simple/2.json
@@ -0,0 +1,30 @@
+{
+ "root_folder": "/tmp/foo/bar",
+ "protocols": [
+ {
+ "name": "protocol",
+ "template": "protocol/1",
+ "views": {
+ "set": {
+ "view": "View"
+ },
+ "set2": {
+ "view": "View2"
+ }
+ }
+ },
+ {
+ "name": "protocol2",
+ "template": "protocol2/1",
+ "views": {
+ "set": {
+ "view": "LargeView"
+ },
+ "set2": {
+ "view": "View2"
+ }
+ }
+ }
+ ],
+ "schema_version": 2
+}
diff --git a/beat/core/test/prefix/databases/simple/2.py b/beat/core/test/prefix/databases/simple/2.py
new file mode 100644
index 0000000000000000000000000000000000000000..e306693b0e8cc617d187cd0143a1c8d6e62559cb
--- /dev/null
+++ b/beat/core/test/prefix/databases/simple/2.py
@@ -0,0 +1,22 @@
+class View:
+ def setup(
+ self,
+ root_folder,
+ outputs,
+ parameters,
+ force_start_index=None,
+ force_end_index=None,
+ ):
+ """Initializes the database"""
+
+ return True
+
+ def done(self):
+ """Should return ``True``, when data is finished"""
+
+ return True
+
+ def next(self):
+ """Loads the next data block on ``outputs``"""
+
+ return True
diff --git a/beat/core/test/prefix/protocoltemplates/large/1.json b/beat/core/test/prefix/protocoltemplates/large/1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6e5978960ee7eee946ea9eb68d7cccf4ad1ed279
--- /dev/null
+++ b/beat/core/test/prefix/protocoltemplates/large/1.json
@@ -0,0 +1,11 @@
+{
+ "schema_version": 1,
+ "sets": [
+ {
+ "name": "data",
+ "outputs": {
+ "out": "user/empty_1d_array_of_integers/1"
+ }
+ }
+ ]
+}
diff --git a/beat/core/test/prefix/protocoltemplates/protocol/1.json b/beat/core/test/prefix/protocoltemplates/protocol/1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1ec082f303681488db3a7c31725e24bcc90e6df2
--- /dev/null
+++ b/beat/core/test/prefix/protocoltemplates/protocol/1.json
@@ -0,0 +1,17 @@
+{
+ "schema_version": 1,
+ "sets": [
+ {
+ "name": "set",
+ "outputs": {
+ "out": "user/single_integer/1"
+ }
+ },
+ {
+ "name": "set2",
+ "outputs": {
+ "out": "user/single_integer/1"
+ }
+ }
+ ]
+}
diff --git a/beat/core/test/prefix/protocoltemplates/protocol2/1.json b/beat/core/test/prefix/protocoltemplates/protocol2/1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1ec082f303681488db3a7c31725e24bcc90e6df2
--- /dev/null
+++ b/beat/core/test/prefix/protocoltemplates/protocol2/1.json
@@ -0,0 +1,17 @@
+{
+ "schema_version": 1,
+ "sets": [
+ {
+ "name": "set",
+ "outputs": {
+ "out": "user/single_integer/1"
+ }
+ },
+ {
+ "name": "set2",
+ "outputs": {
+ "out": "user/single_integer/1"
+ }
+ }
+ ]
+}
diff --git a/beat/core/test/prefix/protocoltemplates/small/1.json b/beat/core/test/prefix/protocoltemplates/small/1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b14901ed5cf2bb9fe324c27cb3fbf6bc1848073b
--- /dev/null
+++ b/beat/core/test/prefix/protocoltemplates/small/1.json
@@ -0,0 +1,11 @@
+{
+ "schema_version": 1,
+ "sets": [
+ {
+ "name": "data",
+ "outputs": {
+ "out": "user/single_integer/1"
+ }
+ }
+ ]
+}
diff --git a/beat/core/test/test_bcp.py b/beat/core/test/test_bcp.py
index 5fe5702e0749cf81f99715d2bb85e0711d71ccf2..10d398d0ef04ffbb141cf290ba1992fc10743457 100644
--- a/beat/core/test/test_bcp.py
+++ b/beat/core/test/test_bcp.py
@@ -322,7 +322,7 @@ class TestBCPDocker(TestBCP):
@classmethod
def setUpClass(cls):
- cls.images_cache = os.path.join(tmp_prefix, "docker_images_cache.json")
+ cls.docker_images_cache = os.path.join(tmp_prefix, "docker_images_cache.json")
cls.host = Host(images_cache=cls.docker_images_cache, raise_on_errors=False)
diff --git a/beat/core/test/test_database.py b/beat/core/test/test_database.py
index 27cd3766f374719493a82db4b7fcbfdd03b9848e..8296534f611b1d890fa149bfa65a67cfc7100ffc 100644
--- a/beat/core/test/test_database.py
+++ b/beat/core/test/test_database.py
@@ -41,15 +41,21 @@ from . import prefix, tmp_prefix
from .utils import cleanup
-@nose.tools.with_setup(teardown=cleanup)
def test_export():
+ for i in range(1, 3):
+ yield export, f"integers_db/{i}"
+ yield export, f"simple/{i}"
+ yield export, f"large/{i}"
+
+
+@nose.tools.with_setup(teardown=cleanup)
+def export(db_name):
- name = "integers_db/1"
- obj = Database(prefix, name)
+ obj = Database(prefix, db_name)
nose.tools.assert_true(obj.valid, "\n * %s" % "\n * ".join(obj.errors))
obj.export(tmp_prefix)
# load from tmp_prefix and validates
- exported = Database(tmp_prefix, name)
+ exported = Database(tmp_prefix, db_name)
nose.tools.assert_true(exported.valid, "\n * %s" % "\n * ".join(exported.errors))
diff --git a/beat/core/test/test_protocoltemplate.py b/beat/core/test/test_protocoltemplate.py
new file mode 100644
index 0000000000000000000000000000000000000000..526a85602e81f87e821c765d666bda27b9686684
--- /dev/null
+++ b/beat/core/test/test_protocoltemplate.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###################################################################################
+# #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are met: #
+# #
+# 1. Redistributions of source code must retain the above copyright notice, this #
+# list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright notice, #
+# this list of conditions and the following disclaimer in the documentation #
+# and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its contributors #
+# may be used to endorse or promote products derived from this software without #
+# specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+###################################################################################
+
+
+import nose.tools
+from ..protocoltemplate import ProtocolTemplate
+
+from . import prefix, tmp_prefix
+from .utils import cleanup
+
+
+def test_export():
+ for protocol_name in [
+ "double",
+ "triple",
+ "two_sets",
+ "labelled",
+ "different_frequencies",
+ ]:
+ yield export, f"{protocol_name}/1"
+
+
+@nose.tools.with_setup(teardown=cleanup)
+def export(protocol_name):
+
+ obj = ProtocolTemplate(prefix, protocol_name)
+ nose.tools.assert_true(obj.valid, "\n * %s" % "\n * ".join(obj.errors))
+
+ obj.export(tmp_prefix)
+
+ # load from tmp_prefix and validates
+ exported = ProtocolTemplate(tmp_prefix, protocol_name)
+ nose.tools.assert_true(exported.valid, "\n * %s" % "\n * ".join(exported.errors))
diff --git a/beat/core/test/test_schema.py b/beat/core/test/test_schema.py
index 8f76da7fa56eb55ba5dc84fcc78dc75be58d7ebe..5940fdbc95e119fd9b32ef79a38bd1ba88addff8 100644
--- a/beat/core/test/test_schema.py
+++ b/beat/core/test/test_schema.py
@@ -37,31 +37,14 @@
from ..schema import load_schema
-def test_common():
-
- load_schema("common")
-
-
-def test_dataformat():
-
- load_schema("dataformat")
-
-
-def test_algorithm():
-
- load_schema("algorithm")
-
-
-def test_database():
-
- load_schema("database")
-
-
-def test_toolchain():
-
- load_schema("toolchain")
-
-
-def test_experiment():
-
- load_schema("experiment")
+def test_load_schema():
+ for item in [
+ "common",
+ "dataformat",
+ "algorithm",
+ "protocoltemplate",
+ "database",
+ "toolchain",
+ "experiment",
+ ]:
+ yield load_schema, item
diff --git a/beat/core/test/test_worker.py b/beat/core/test/test_worker.py
index 2d8196a1cae0b4de6654810138fb61b3ac4f918d..6249c4707d0edfa4a082ade6f6d253e22043686c 100644
--- a/beat/core/test/test_worker.py
+++ b/beat/core/test/test_worker.py
@@ -46,6 +46,9 @@ import queue
from time import time
from time import sleep
+from ddt import ddt
+from ddt import idata
+
from ..scripts import worker
from ..worker import WorkerController
from ..database import Database
@@ -66,6 +69,8 @@ PORT = find_free_port()
# ----------------------------------------------------------
+DATABASES = [f"integers_db/{i}" for i in range(1, 3)]
+
CONFIGURATION1 = {
"queue": "queue",
@@ -133,6 +138,19 @@ CONFIGURATION2 = {
# ----------------------------------------------------------
+def prepare_database(db_name):
+ CONFIGURATION1["inputs"]["in"]["database"] = db_name
+ CONFIGURATION2["inputs"]["in"]["database"] = db_name
+
+ for _, input_cfg in CONFIGURATION1["inputs"].items():
+ database = Database(prefix, input_cfg["database"])
+ view = database.view(input_cfg["protocol"], input_cfg["set"])
+ view.index(os.path.join(tmp_prefix, input_cfg["path"]))
+
+
+# ----------------------------------------------------------
+
+
class ControllerProcess(multiprocessing.Process):
def __init__(self, queue):
super(ControllerProcess, self).__init__()
@@ -271,12 +289,6 @@ class TestWorkerBase(unittest.TestCase):
self.assertTrue(name not in self.controller.workers)
- def prepare_databases(self, configuration):
- for _, input_cfg in configuration["inputs"].items():
- database = Database(prefix, input_cfg["database"])
- view = database.view(input_cfg["protocol"], input_cfg["set"])
- view.index(os.path.join(tmp_prefix, input_cfg["path"]))
-
# ----------------------------------------------------------
@@ -361,6 +373,7 @@ class TestConnection(TestWorkerBase):
# ----------------------------------------------------------
+@ddt
class TestOneWorker(TestWorkerBase):
def setUp(self):
super(TestOneWorker, self).setUp()
@@ -370,8 +383,6 @@ class TestOneWorker(TestWorkerBase):
self.wait_for_worker_connection(WORKER1)
- self.prepare_databases(CONFIGURATION1)
-
def _wait(self, max=200):
message = None
nb = 0
@@ -398,14 +409,20 @@ class TestOneWorker(TestWorkerBase):
self.assertEqual(result["status"], 0)
- def test_success(self):
+ @idata(DATABASES)
+ def test_success(self, db_name):
+ prepare_database(db_name)
+
self.controller.execute(WORKER1, 1, CONFIGURATION1)
message = self._wait()
self._check_done(message, WORKER1, 1)
- def test_processing_error(self):
+ @idata(DATABASES)
+ def test_processing_error(self, db_name):
+ prepare_database(db_name)
+
config = dict(CONFIGURATION1)
config["algorithm"] = "legacy/process_crash/1"
@@ -424,7 +441,10 @@ class TestOneWorker(TestWorkerBase):
self.assertEqual(result["status"], 1)
self.assertTrue("a = b" in result["user_error"])
- def test_error_unknown_algorithm(self):
+ @idata(DATABASES)
+ def test_error_unknown_algorithm(self, db_name):
+ prepare_database(db_name)
+
config = dict(CONFIGURATION1)
config["algorithm"] = "user/unknown/1"
@@ -439,7 +459,10 @@ class TestOneWorker(TestWorkerBase):
self.assertEqual(job_id, 1)
self.assertTrue(len(data) > 0)
- def test_error_syntax_error(self):
+ @idata(DATABASES)
+ def test_error_syntax_error(self, db_name):
+ prepare_database(db_name)
+
config = dict(CONFIGURATION1)
config["algorithm"] = "legacy/syntax_error/1"
@@ -454,7 +477,10 @@ class TestOneWorker(TestWorkerBase):
self.assertEqual(job_id, 1)
self.assertTrue(len(data) > 0)
- def test_multiple_jobs(self):
+ @idata(DATABASES)
+ def test_multiple_jobs(self, db_name):
+ prepare_database(db_name)
+
config = dict(CONFIGURATION1)
config["algorithm"] = "user/integers_echo_slow/1"
@@ -467,7 +493,10 @@ class TestOneWorker(TestWorkerBase):
message = self._wait()
self._check_done(message, WORKER1, 2)
- def test_reuse(self):
+ @idata(DATABASES)
+ def test_reuse(self, db_name):
+ prepare_database(db_name)
+
self.controller.execute(WORKER1, 1, CONFIGURATION1)
message = self._wait()
self._check_done(message, WORKER1, 1)
@@ -476,7 +505,10 @@ class TestOneWorker(TestWorkerBase):
message = self._wait()
self._check_done(message, WORKER1, 2)
- def test_cancel(self):
+ @idata(DATABASES)
+ def test_cancel(self, db_name):
+ prepare_database(db_name)
+
config = dict(CONFIGURATION1)
config["algorithm"] = "user/integers_echo_slow/1"
@@ -508,6 +540,7 @@ class TestOneWorker(TestWorkerBase):
# ----------------------------------------------------------
+@ddt
class TestTwoWorkers(TestWorkerBase):
def setUp(self):
self.tearDown() # In case another test failed badly during its setUp()
@@ -520,7 +553,9 @@ class TestTwoWorkers(TestWorkerBase):
self.wait_for_worker_connection(WORKER1)
self.wait_for_worker_connection(WORKER2)
- def _test_success_one_worker(self, worker_name):
+ def _test_success_one_worker(self, worker_name, db_name):
+ prepare_database(db_name)
+
self.controller.execute(worker_name, 1, CONFIGURATION1)
message = None
@@ -538,13 +573,16 @@ class TestTwoWorkers(TestWorkerBase):
self.assertEqual(result["status"], 0)
- def test_success_worker1(self):
- self._test_success_one_worker(WORKER1)
+ @idata(DATABASES)
+ def test_success_worker1(self, db_name):
+ self._test_success_one_worker(WORKER1, db_name)
- def test_success_worker2(self):
- self._test_success_one_worker(WORKER2)
+ @idata(DATABASES)
+ def test_success_worker2(self, db_name):
+ self._test_success_one_worker(WORKER2, db_name)
- def test_success_both_workers(self):
+ @idata(DATABASES)
+ def test_success_both_workers(self, db_name):
def _check(worker, status, job_id, data):
self.assertEqual(status, WorkerController.DONE)
@@ -557,6 +595,8 @@ class TestTwoWorkers(TestWorkerBase):
result = json.loads(data[0])
self.assertEqual(result["status"], 0)
+ prepare_database(db_name)
+
self.controller.execute(WORKER1, 1, CONFIGURATION1)
self.controller.execute(WORKER2, 2, CONFIGURATION2)
diff --git a/beat/core/test/utils.py b/beat/core/test/utils.py
index d566048f680d841d66153f9aac59edbb8c06d29d..ab2191035141c925db00bbdba738b2f24ec65c12 100644
--- a/beat/core/test/utils.py
+++ b/beat/core/test/utils.py
@@ -48,8 +48,8 @@ import docker
# Images used for docker-enabled tests within this and other BEAT packages
DOCKER_TEST_IMAGES = {
- "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
- "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
+ "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r5",
+ "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r5",
"docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
"docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
}
diff --git a/conda/meta.yaml b/conda/meta.yaml
index fc13b17cd6072d26c4df7947ba645253dd1ad210..b02b796e30325e1bba00d38dc3a9a079a1c87991 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -37,7 +37,7 @@ requirements:
- pyzmq
- simplejson
- six
- - beat.backend.python >=1.6.0a0
+ - beat.backend.python >=1.7.0b0
- matplotlib
- pillow
@@ -46,6 +46,7 @@ test:
- bob-devel {{ bob_devel }}.*
- beat-devel {{ beat_devel }}.*
- bob.extension
+ - ddt
- nose
- coverage
- sphinx