Skip to content
GitLab
Commit 9af90457 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch '64_add_optionnal_environment_entry_to_database' into 'master' 

Add optionnal environment entry to database

See merge request !122
parents 72737fdd 7b28f3aa
Pipeline #40697 passed with stages
in 17 minutes and 59 seconds
Hide whitespace changes
Inline Side-by-side
beat/core/dock.py
View file @ 9af90457
......@@ -57,6 +57,8 @@ from packaging import version
from beat.core import stats
from .utils import build_env_name
logger = logging.getLogger(__name__)
......@@ -91,9 +93,10 @@ class Host(object):
) = self._discover_environments_using_labels()
if not self.db_environments and not self.processing_environments:
self.processing_environments, self.db_environments = (
self._discover_environments_using_describe()
)
(
self.processing_environments,
self.db_environments,
) = self._discover_environments_using_describe()
# (If necessary) Save the known infos about the images
if self.images_cache_filename is not None:
......@@ -131,6 +134,12 @@ class Host(object):
return attrs["image"]
def dbenv2docker(self, key):
"""Returns a nice docker image name given a BEAT database environment key"""
attrs = self.db_environments[key]
return attrs["image"]
def teardown(self):
for container in self.containers:
self.rm(container)
......@@ -312,7 +321,7 @@ class Host(object):
logger.debug("Description not found for", image)
continue
key = description["name"] + " (" + description["version"] + ")"
key = build_env_name(description)
if "databases" in description:
if (key in db_environments) and not _must_replace(
......@@ -400,7 +409,7 @@ class Host(object):
continue
image_info = _parse_image_info(image)
key = "{} ({})".format(image_info["name"], image_info["version"])
key = build_env_name(image_info)
image_name = image_info["image"]
if key in environments:
......
beat/core/execution/docker.py
View file @ 9af90457
......@@ -48,6 +48,8 @@ import logging
import requests
import simplejson as json
from collections import namedtuple
from beat.backend.python.execution import MessageHandler
from beat.backend.python.data import getAllFilenames
......@@ -188,33 +190,79 @@ class DockerExecutor(RemoteExecutor):
# Modify the paths to the databases in the dumped configuration files
root_folder = os.path.join(databases_configuration_path, "prefix", "databases")
database_paths = {}
DatabaseInfo = namedtuple("DatabaseInfo", ["path", "environment"])
databases_infos = {}
for db_name in self.databases.keys():
for db_name, db_object, in self.databases.items():
json_path = os.path.join(root_folder, db_name + ".json")
with open(json_path, "r") as f:
db_data = json.load(f)
database_paths[db_name] = db_data["root_folder"]
db_data["root_folder"] = os.path.join("/databases", db_name)
system_path = db_data["root_folder"]
container_path = os.path.join("/databases", db_name)
db_data["root_folder"] = container_path
with open(json_path, "w") as f:
json.dump(db_data, f, indent=4)
# Determine the docker image to use for the databases
try:
databases_environment = self.host.db2docker(database_paths.keys())
except Exception:
raise RuntimeError(
"No environment found for the databases `%s' "
"- available environments are %s"
% (
", ".join(database_paths.keys()),
", ".join(self.host.db_environments.keys()),
)
databases_infos[db_name] = DatabaseInfo(
system_path, utils.build_env_name(db_object.environment)
)
databases_environment = None
requesting_environments = {
name: info
for name, info in databases_infos.items()
if info.environment is not None
}
if requesting_environments:
if len(requesting_environments) != len(self.databases):
raise RuntimeError(
"Selected databases ({}) are not all providing"
" an environment.".format(list(self.databases.keys()))
)
requested_environments = {
info.environment
for info in requesting_environments.values()
if info.environment is not None
}
if len(requested_environments) > 1:
raise RuntimeError(
"Selected databases ({}) are requesting different environments,"
"only one is supported".format(list(requesting_environments.keys()))
)
# All databases are requesting the same environment
db_environment = next(iter(requested_environments))
try:
databases_environment = self.host.dbenv2docker(db_environment)
except Exception:
raise RuntimeError(
"Environment {} not found - available environments are {}".format(
db_environment, list(self.host.db_environments.keys())
)
)
if not databases_environment:
# Determine the docker image to use for the databases
database_list = databases_infos.keys()
try:
databases_environment = self.host.db2docker(database_list)
except Exception:
raise RuntimeError(
"No environment found for the databases `%s' "
"- available environments are %s"
% (
", ".join(database_list),
", ".join(self.host.db_environments.keys()),
)
)
# Creation of the container
# Note: we only support one databases image loaded at the same time
database_port = utils.find_free_port()
......@@ -244,8 +292,8 @@ class DockerExecutor(RemoteExecutor):
)
databases_info.add_volume(self.cache, self.CONTAINER_CACHE_PATH)
for db_name, db_path in database_paths.items():
databases_info.add_volume(db_path, os.path.join("/databases", db_name))
for db_name, db_info in databases_infos.items():
databases_info.add_volume(db_info.path, os.path.join("/databases", db_name))
# Start the container
while True:
......@@ -392,7 +440,7 @@ class DockerExecutor(RemoteExecutor):
)
# Determine the docker image to use for the processing
processing_environment = "%(name)s (%(version)s)" % self.data["environment"]
processing_environment = utils.build_env_name(self.data["environment"])
if processing_environment not in self.host:
raise RuntimeError(
"Environment `%s' is not available on docker "
......
beat/core/schema/database/1.json
View file @ 9af90457
......@@ -19,6 +19,19 @@
"items": { "$ref": "#/definitions/protocol" }
},
"environment": {
"type": "object",
"properties": {
"name": { "type": "string" },
"version": { "type": "string" }
},
"required": [
"name",
"version"
],
"additionalProperties": false
},
"description": { "$ref": "../common/1.json#/definitions/description" },
"schema_version": { "$ref": "../common/1.json#/definitions/version" }
......
beat/core/schema/database/2.json
View file @ 9af90457
......@@ -21,6 +21,19 @@
"description": { "$ref": "../common/1.json#/definitions/description" },
"environment": {
"type": "object",
"properties": {
"name": { "type": "string" },
"version": { "type": "string" }
},
"required": [
"name",
"version"
],
"additionalProperties": false
},
"schema_version": { "const": 2 }
},
......
beat/core/test/prefix/databases/integers_db_invalid_env/1.json 0 → 100644
View file @ 9af90457
{
"root_folder": "/tmp/path/not/set",
"environment": {
"name": "Does not exist",
"version": "1.4.0"
},
"protocols": [
{
"name": "double",
"template": "double",
"sets": [
{
"name": "double",
"template": "double",
"view": "Double",
"outputs": {
"a": "user/single_integer/1",
"b": "user/single_integer/1",
"sum": "user/single_integer/1"
}
}
]
},
{
"name": "triple",
"template": "triple",
"sets": [
{
"name": "triple",
"view": "Triple",
"template": "triple",
"outputs": {
"a": "user/single_integer/1",
"b": "user/single_integer/1",
"c": "user/single_integer/1",
"sum": "user/single_integer/1"
}
}
]
},
{
"name": "two_sets",
"template": "two_sets",
"sets": [
{
"name": "double",
"template": "double",
"view": "Double",
"outputs": {
"a": "user/single_integer/1",
"b": "user/single_integer/1",
"sum": "user/single_integer/1"
}
},
{
"name": "triple",
"template": "triple",
"view": "Triple",
"outputs": {
"a": "user/single_integer/1",
"b": "user/single_integer/1",
"c": "user/single_integer/1",
"sum": "user/single_integer/1"
}
}
]
},
{
"name": "labelled",
"template": "labelled",
"sets": [
{
"name": "labelled",
"template": "labelled",
"view": "Labelled",
"outputs": {
"value": "user/single_integer/1",
"label": "user/single_string/1"
}
}
]
},
{
"name": "different_frequencies",
"template": "different_frequencies",
"sets": [
{
"name": "double",
"template": "double",
"view": "DifferentFrequencies",
"outputs": {
"a": "user/single_integer/1",
"b": "user/single_integer/1"
}
}
]
}
]
}
beat/core/test/prefix/databases/integers_db_invalid_env/1.py 0 → 100755
View file @ 9af90457
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import numpy
from collections import namedtuple
from beat.backend.python.database import View
class Double(View):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["a", "b", "sum"])
return [
Entry(1, 10, 11),
Entry(2, 20, 22),
Entry(3, 30, 33),
Entry(4, 40, 44),
Entry(5, 50, 55),
Entry(6, 60, 66),
Entry(7, 70, 77),
Entry(8, 80, 88),
Entry(9, 90, 99),
]
def get(self, output, index):
obj = self.objs[index]
if output == "a":
return {"value": numpy.int32(obj.a)}
elif output == "b":
return {"value": numpy.int32(obj.b)}
elif output == "sum":
return {"value": numpy.int32(obj.sum)}
elif output == "class":
return {"value": numpy.int32(obj.cls)}
# ----------------------------------------------------------
class Triple(View):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["a", "b", "c", "sum"])
return [
Entry(1, 10, 100, 111),
Entry(2, 20, 200, 222),
Entry(3, 30, 300, 333),
Entry(4, 40, 400, 444),
Entry(5, 50, 500, 555),
Entry(6, 60, 600, 666),
Entry(7, 70, 700, 777),
Entry(8, 80, 800, 888),
Entry(9, 90, 900, 999),
]
def get(self, output, index):
obj = self.objs[index]
if output == "a":
return {"value": numpy.int32(obj.a)}
elif output == "b":
return {"value": numpy.int32(obj.b)}
elif output == "c":
return {"value": numpy.int32(obj.c)}
elif output == "sum":
return {"value": numpy.int32(obj.sum)}
# ----------------------------------------------------------
class Labelled(View):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["label", "value"])
return [
Entry("A", 1),
Entry("A", 2),
Entry("A", 3),
Entry("A", 4),
Entry("A", 5),
Entry("B", 10),
Entry("B", 20),
Entry("B", 30),
Entry("B", 40),
Entry("B", 50),
Entry("C", 100),
Entry("C", 200),
Entry("C", 300),
Entry("C", 400),
Entry("C", 500),
]
def get(self, output, index):
obj = self.objs[index]
if output == "label":
return {"value": obj.label}
elif output == "value":
return {"value": numpy.int32(obj.value)}
# ----------------------------------------------------------
class DifferentFrequencies(View):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["a", "b"])
return [
Entry(1, 10),
Entry(1, 20),
Entry(1, 30),
Entry(1, 40),
Entry(2, 50),
Entry(2, 60),
Entry(2, 70),
Entry(2, 80),
]
def get(self, output, index):
obj = self.objs[index]
if output == "a":
return {"value": numpy.int32(obj.a)}
elif output == "b":
return {"value": numpy.int32(obj.b)}
beat/core/test/prefix/databases/integers_db_invalid_env/2.json 0 → 100644
View file @ 9af90457
{
"schema_version": 2,
"environment": {
"name": "Does not exist",
"version": "1.4.0"
},
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
"template": "double/1",
"views": {
"double": {
"view": "Double"
}
}
},
{
"name": "triple",
"template": "triple/1",
"views": {
"triple": {
"view": "Triple"
}
}
},
{
"name": "two_sets",
"template": "two_sets/1",
"views": {
"double": {
"view": "Double"
},
"triple": {
"view": "Triple"
}
}
},
{
"name": "labelled",
"template": "labelled/1",
"views": {
"labelled": {
"view": "Labelled"
}
}
},
{
"name": "different_frequencies",
"template": "different_frequencies/1",
"views": {
"double" : {
"view": "DifferentFrequencies"
}
}
}
]
}
beat/core/test/prefix/databases/integers_db_invalid_env/2.py 0 → 100644
View file @ 9af90457
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import numpy
from collections import namedtuple
from beat.backend.python.database import View
class Double(View):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["a", "b", "sum"])