Skip to content

beat.core
beat.core
Commit 9af90457 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files
Options

Merge branch '64_add_optionnal_environment_entry_to_database' into 'master' 

Add optionnal environment entry to database

See merge request !122
parents 72737fdd 7b28f3aa
Pipeline #40697 passed with stages
      in 17 minutes and 59 seconds
      Hide whitespace changes
      Inline Side-by-side
      Showing with 1635 additions and 25 deletions
      beat/core/dock.py
      View file @ 9af90457
      ......@@ -57,6 +57,8 @@ from packaging import version
      from beat.core import stats
      from .utils import build_env_name
      logger = logging.getLogger(__name__)
      ......@@ -91,9 +93,10 @@ class Host(object):
      ) = self._discover_environments_using_labels()
      if not self.db_environments and not self.processing_environments:
      self.processing_environments, self.db_environments = (
      self._discover_environments_using_describe()
      )
      (
      self.processing_environments,
      self.db_environments,
      ) = self._discover_environments_using_describe()
      # (If necessary) Save the known infos about the images
      if self.images_cache_filename is not None:
      ......@@ -131,6 +134,12 @@ class Host(object):
      return attrs["image"]
      def dbenv2docker(self, key):
      """Returns a nice docker image name given a BEAT database environment key"""
      attrs = self.db_environments[key]
      return attrs["image"]
      def teardown(self):
      for container in self.containers:
      self.rm(container)
      ......@@ -312,7 +321,7 @@ class Host(object):
      logger.debug("Description not found for", image)
      continue
      key = description["name"] + " (" + description["version"] + ")"
      key = build_env_name(description)
      if "databases" in description:
      if (key in db_environments) and not _must_replace(
      ......@@ -400,7 +409,7 @@ class Host(object):
      continue
      image_info = _parse_image_info(image)
      key = "{} ({})".format(image_info["name"], image_info["version"])
      key = build_env_name(image_info)
      image_name = image_info["image"]
      if key in environments:
      ......
      beat/core/execution/docker.py
      View file @ 9af90457
      ......@@ -48,6 +48,8 @@ import logging
      import requests
      import simplejson as json
      from collections import namedtuple
      from beat.backend.python.execution import MessageHandler
      from beat.backend.python.data import getAllFilenames
      ......@@ -188,33 +190,79 @@ class DockerExecutor(RemoteExecutor):
      # Modify the paths to the databases in the dumped configuration files
      root_folder = os.path.join(databases_configuration_path, "prefix", "databases")
      database_paths = {}
      DatabaseInfo = namedtuple("DatabaseInfo", ["path", "environment"])
      databases_infos = {}
      for db_name in self.databases.keys():
      for db_name, db_object, in self.databases.items():
      json_path = os.path.join(root_folder, db_name + ".json")
      with open(json_path, "r") as f:
      db_data = json.load(f)
      database_paths[db_name] = db_data["root_folder"]
      db_data["root_folder"] = os.path.join("/databases", db_name)
      system_path = db_data["root_folder"]
      container_path = os.path.join("/databases", db_name)
      db_data["root_folder"] = container_path
      with open(json_path, "w") as f:
      json.dump(db_data, f, indent=4)
      # Determine the docker image to use for the databases
      try:
      databases_environment = self.host.db2docker(database_paths.keys())
      except Exception:
      raise RuntimeError(
      "No environment found for the databases `%s' "
      "- available environments are %s"
      % (
      ", ".join(database_paths.keys()),
      ", ".join(self.host.db_environments.keys()),
      )
      databases_infos[db_name] = DatabaseInfo(
      system_path, utils.build_env_name(db_object.environment)
      )
      databases_environment = None
      requesting_environments = {
      name: info
      for name, info in databases_infos.items()
      if info.environment is not None
      }
      if requesting_environments:
      if len(requesting_environments) != len(self.databases):
      raise RuntimeError(
      "Selected databases ({}) are not all providing"
      " an environment.".format(list(self.databases.keys()))
      )
      requested_environments = {
      info.environment
      for info in requesting_environments.values()
      if info.environment is not None
      }
      if len(requested_environments) > 1:
      raise RuntimeError(
      "Selected databases ({}) are requesting different environments,"
      "only one is supported".format(list(requesting_environments.keys()))
      )
      # All databases are requesting the same environment
      db_environment = next(iter(requested_environments))
      try:
      databases_environment = self.host.dbenv2docker(db_environment)
      except Exception:
      raise RuntimeError(
      "Environment {} not found - available environments are {}".format(
      db_environment, list(self.host.db_environments.keys())
      )
      )
      if not databases_environment:
      # Determine the docker image to use for the databases
      database_list = databases_infos.keys()
      try:
      databases_environment = self.host.db2docker(database_list)
      except Exception:
      raise RuntimeError(
      "No environment found for the databases `%s' "
      "- available environments are %s"
      % (
      ", ".join(database_list),
      ", ".join(self.host.db_environments.keys()),
      )
      )
      # Creation of the container
      # Note: we only support one databases image loaded at the same time
      database_port = utils.find_free_port()
      ......@@ -244,8 +292,8 @@ class DockerExecutor(RemoteExecutor):
      )
      databases_info.add_volume(self.cache, self.CONTAINER_CACHE_PATH)
      for db_name, db_path in database_paths.items():
      databases_info.add_volume(db_path, os.path.join("/databases", db_name))
      for db_name, db_info in databases_infos.items():
      databases_info.add_volume(db_info.path, os.path.join("/databases", db_name))
      # Start the container
      while True:
      ......@@ -392,7 +440,7 @@ class DockerExecutor(RemoteExecutor):
      )
      # Determine the docker image to use for the processing
      processing_environment = "%(name)s (%(version)s)" % self.data["environment"]
      processing_environment = utils.build_env_name(self.data["environment"])
      if processing_environment not in self.host:
      raise RuntimeError(
      "Environment `%s' is not available on docker "
      ......
      beat/core/schema/database/1.json
      View file @ 9af90457
      ......@@ -19,6 +19,19 @@
      "items": { "$ref": "#/definitions/protocol" }
      },
      "environment": {
      "type": "object",
      "properties": {
      "name": { "type": "string" },
      "version": { "type": "string" }
      },
      "required": [
      "name",
      "version"
      ],
      "additionalProperties": false
      },
      "description": { "$ref": "../common/1.json#/definitions/description" },
      "schema_version": { "$ref": "../common/1.json#/definitions/version" }
      ......
      beat/core/schema/database/2.json
      View file @ 9af90457
      ......@@ -21,6 +21,19 @@
      "description": { "$ref": "../common/1.json#/definitions/description" },
      "environment": {
      "type": "object",
      "properties": {
      "name": { "type": "string" },
      "version": { "type": "string" }
      },
      "required": [
      "name",
      "version"
      ],
      "additionalProperties": false
      },
      "schema_version": { "const": 2 }
      },
      ......
      beat/core/test/prefix/databases/integers_db_invalid_env/1.json 0 → 100644
      View file @ 9af90457
      {
      "root_folder": "/tmp/path/not/set",
      "environment": {
      "name": "Does not exist",
      "version": "1.4.0"
      },
      "protocols": [
      {
      "name": "double",
      "template": "double",
      "sets": [
      {
      "name": "double",
      "template": "double",
      "view": "Double",
      "outputs": {
      "a": "user/single_integer/1",
      "b": "user/single_integer/1",
      "sum": "user/single_integer/1"
      }
      }
      ]
      },
      {
      "name": "triple",
      "template": "triple",
      "sets": [
      {
      "name": "triple",
      "view": "Triple",
      "template": "triple",
      "outputs": {
      "a": "user/single_integer/1",
      "b": "user/single_integer/1",
      "c": "user/single_integer/1",
      "sum": "user/single_integer/1"
      }
      }
      ]
      },
      {
      "name": "two_sets",
      "template": "two_sets",
      "sets": [
      {
      "name": "double",
      "template": "double",
      "view": "Double",
      "outputs": {
      "a": "user/single_integer/1",
      "b": "user/single_integer/1",
      "sum": "user/single_integer/1"
      }
      },
      {
      "name": "triple",
      "template": "triple",
      "view": "Triple",
      "outputs": {
      "a": "user/single_integer/1",
      "b": "user/single_integer/1",
      "c": "user/single_integer/1",
      "sum": "user/single_integer/1"
      }
      }
      ]
      },
      {
      "name": "labelled",
      "template": "labelled",
      "sets": [
      {
      "name": "labelled",
      "template": "labelled",
      "view": "Labelled",
      "outputs": {
      "value": "user/single_integer/1",
      "label": "user/single_string/1"
      }
      }
      ]
      },
      {
      "name": "different_frequencies",
      "template": "different_frequencies",
      "sets": [
      {
      "name": "double",
      "template": "double",
      "view": "DifferentFrequencies",
      "outputs": {
      "a": "user/single_integer/1",
      "b": "user/single_integer/1"
      }
      }
      ]
      }
      ]
      }
      beat/core/test/prefix/databases/integers_db_invalid_env/1.py 0 → 100755
      View file @ 9af90457
      #!/usr/bin/env python
      # vim: set fileencoding=utf-8 :
      ###################################################################################
      # #
      # Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
      # Contact: beat.support@idiap.ch #
      # #
      # Redistribution and use in source and binary forms, with or without #
      # modification, are permitted provided that the following conditions are met: #
      # #
      # 1. Redistributions of source code must retain the above copyright notice, this #
      # list of conditions and the following disclaimer. #
      # #
      # 2. Redistributions in binary form must reproduce the above copyright notice, #
      # this list of conditions and the following disclaimer in the documentation #
      # and/or other materials provided with the distribution. #
      # #
      # 3. Neither the name of the copyright holder nor the names of its contributors #
      # may be used to endorse or promote products derived from this software without #
      # specific prior written permission. #
      # #
      # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
      # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
      # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
      # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
      # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
      # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
      # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
      # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
      # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
      # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
      # #
      ###################################################################################
      import numpy
      from collections import namedtuple
      from beat.backend.python.database import View
      class Double(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["a", "b", "sum"])
      return [
      Entry(1, 10, 11),
      Entry(2, 20, 22),
      Entry(3, 30, 33),
      Entry(4, 40, 44),
      Entry(5, 50, 55),
      Entry(6, 60, 66),
      Entry(7, 70, 77),
      Entry(8, 80, 88),
      Entry(9, 90, 99),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "a":
      return {"value": numpy.int32(obj.a)}
      elif output == "b":
      return {"value": numpy.int32(obj.b)}
      elif output == "sum":
      return {"value": numpy.int32(obj.sum)}
      elif output == "class":
      return {"value": numpy.int32(obj.cls)}
      # ----------------------------------------------------------
      class Triple(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["a", "b", "c", "sum"])
      return [
      Entry(1, 10, 100, 111),
      Entry(2, 20, 200, 222),
      Entry(3, 30, 300, 333),
      Entry(4, 40, 400, 444),
      Entry(5, 50, 500, 555),
      Entry(6, 60, 600, 666),
      Entry(7, 70, 700, 777),
      Entry(8, 80, 800, 888),
      Entry(9, 90, 900, 999),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "a":
      return {"value": numpy.int32(obj.a)}
      elif output == "b":
      return {"value": numpy.int32(obj.b)}
      elif output == "c":
      return {"value": numpy.int32(obj.c)}
      elif output == "sum":
      return {"value": numpy.int32(obj.sum)}
      # ----------------------------------------------------------
      class Labelled(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["label", "value"])
      return [
      Entry("A", 1),
      Entry("A", 2),
      Entry("A", 3),
      Entry("A", 4),
      Entry("A", 5),
      Entry("B", 10),
      Entry("B", 20),
      Entry("B", 30),
      Entry("B", 40),
      Entry("B", 50),
      Entry("C", 100),
      Entry("C", 200),
      Entry("C", 300),
      Entry("C", 400),
      Entry("C", 500),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "label":
      return {"value": obj.label}
      elif output == "value":
      return {"value": numpy.int32(obj.value)}
      # ----------------------------------------------------------
      class DifferentFrequencies(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["a", "b"])
      return [
      Entry(1, 10),
      Entry(1, 20),
      Entry(1, 30),
      Entry(1, 40),
      Entry(2, 50),
      Entry(2, 60),
      Entry(2, 70),
      Entry(2, 80),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "a":
      return {"value": numpy.int32(obj.a)}
      elif output == "b":
      return {"value": numpy.int32(obj.b)}
      beat/core/test/prefix/databases/integers_db_invalid_env/2.json 0 → 100644
      View file @ 9af90457
      {
      "schema_version": 2,
      "environment": {
      "name": "Does not exist",
      "version": "1.4.0"
      },
      "root_folder": "/tmp/path/not/set",
      "protocols": [
      {
      "name": "double",
      "template": "double/1",
      "views": {
      "double": {
      "view": "Double"
      }
      }
      },
      {
      "name": "triple",
      "template": "triple/1",
      "views": {
      "triple": {
      "view": "Triple"
      }
      }
      },
      {
      "name": "two_sets",
      "template": "two_sets/1",
      "views": {
      "double": {
      "view": "Double"
      },
      "triple": {
      "view": "Triple"
      }
      }
      },
      {
      "name": "labelled",
      "template": "labelled/1",
      "views": {
      "labelled": {
      "view": "Labelled"
      }
      }
      },
      {
      "name": "different_frequencies",
      "template": "different_frequencies/1",
      "views": {
      "double" : {
      "view": "DifferentFrequencies"
      }
      }
      }
      ]
      }
      beat/core/test/prefix/databases/integers_db_invalid_env/2.py 0 → 100644
      View file @ 9af90457
      #!/usr/bin/env python
      # vim: set fileencoding=utf-8 :
      ###################################################################################
      # #
      # Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
      # Contact: beat.support@idiap.ch #
      # #
      # Redistribution and use in source and binary forms, with or without #
      # modification, are permitted provided that the following conditions are met: #
      # #
      # 1. Redistributions of source code must retain the above copyright notice, this #
      # list of conditions and the following disclaimer. #
      # #
      # 2. Redistributions in binary form must reproduce the above copyright notice, #
      # this list of conditions and the following disclaimer in the documentation #
      # and/or other materials provided with the distribution. #
      # #
      # 3. Neither the name of the copyright holder nor the names of its contributors #
      # may be used to endorse or promote products derived from this software without #
      # specific prior written permission. #
      # #
      # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
      # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
      # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
      # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
      # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
      # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
      # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
      # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
      # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
      # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
      # #
      ###################################################################################
      import numpy
      from collections import namedtuple
      from beat.backend.python.database import View
      class Double(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["a", "b", "sum"])
      return [
      Entry(1, 10, 11),
      Entry(2, 20, 22),
      Entry(3, 30, 33),
      Entry(4, 40, 44),
      Entry(5, 50, 55),
      Entry(6, 60, 66),
      Entry(7, 70, 77),
      Entry(8, 80, 88),
      Entry(9, 90, 99),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "a":
      return {"value": numpy.int32(obj.a)}
      elif output == "b":
      return {"value": numpy.int32(obj.b)}
      elif output == "sum":
      return {"value": numpy.int32(obj.sum)}
      elif output == "class":
      return {"value": numpy.int32(obj.cls)}
      # ----------------------------------------------------------
      class Triple(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["a", "b", "c", "sum"])
      return [
      Entry(1, 10, 100, 111),
      Entry(2, 20, 200, 222),
      Entry(3, 30, 300, 333),
      Entry(4, 40, 400, 444),
      Entry(5, 50, 500, 555),
      Entry(6, 60, 600, 666),
      Entry(7, 70, 700, 777),
      Entry(8, 80, 800, 888),
      Entry(9, 90, 900, 999),
      ]
      def get(self, output, index):
      obj = self.objs[index]
      if output == "a":
      return {"value": numpy.int32(obj.a)}
      elif output == "b":
      return {"value": numpy.int32(obj.b)}
      elif output == "c":
      return {"value": numpy.int32(obj.c)}
      elif output == "sum":
      return {"value": numpy.int32(obj.sum)}
      # ----------------------------------------------------------
      class Labelled(View):
      def index(self, root_folder, parameters):
      Entry = namedtuple("Entry", ["label", "value"])
      return [
      Entry("A", 1),
      Entry("A", 2),
      Entry("A", 3),
      Entry("A", 4),
      Entry("A", 5),
      Entry("B", 10),
      Entry("B", 20),
      Entry("B", 30),
      Entry("B", 40),
      Entry("B", 50),
      Entry("C", 100),
      Entry("C", 200),
      Entry("C", 300),
      Entry("C", 400),