Use a global prefix to keep things around, add an iris test

beat/backend/python/config.py

@@ -99,7 +99,7 @@ def set_config(**kwargs):
     """
     supported_keys = set(DEFAULTS.keys())
     set_keys = set(kwargs.keys())
-    if set_keys not in supported_keys:
+    if not set_keys.issubset(supported_keys):
         raise ValueError(
             f"Only {supported_keys} are valid configurations. "
             f"Got these extra values: {set_keys - supported_keys}"
@@ -137,14 +137,14 @@ def config_context(**new_config):
     """
     old_config = get_config().copy()
     # also backup prefix
-    old_prefix = Prefix().copy()
+    prefix = Prefix()
+    old_prefix = prefix.copy()
     set_config(**new_config)
 
     try:
         yield
     finally:
         set_config(**old_config)
-        prefix = Prefix()
         prefix.clear()
         prefix.update(old_prefix)
 
@@ -154,7 +154,6 @@ def config_context(**new_config):
 
 class Singleton(type):
     """A Singleton metaclass
-    The singleton class calls the __init__ method each time the instance is requested.
     From: https://stackoverflow.com/a/6798042/1286165
     """
 
@@ -167,11 +166,55 @@ class Singleton(type):
 
 
 class Prefix(dict, metaclass=Singleton):
-    def __init__(self, path=None, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    pass
 
 
 class PrefixMeta(type):
+    # cache instances when __init__ is called
+    def __call__(cls, *args, **kwargs):
+        instance = super().__call__(*args, **kwargs)
+
+        folder = f"{cls.asset_folder}/{instance.name}"
+        prefix = Prefix()
+        prefix[folder] = instance
+        print(f"caching {cls.__name__}/{instance.name} in __init__ calls")
+
+        return instance
+
+    def __init__(cls, name, bases, clsdict):
+        # cache instances when new is called
+        if "new" in clsdict:
+
+            def caching_new(*args, **kwargs):
+                instance = clsdict["new"].__func__(cls, *args, **kwargs)
+                folder = f"{cls.asset_folder}/{instance.name}"
+                prefix = Prefix()
+                prefix[folder] = instance
+                print(f"caching {cls.__name__}/{instance.name} in new calls")
+                return instance
+
+            setattr(cls, "new", caching_new)
+
+        # change the key of instance when its name is changed
+        if "name" in clsdict:
+            name_property = clsdict["name"]
+            actual_fset = name_property.fset
+
+            def updating_fset(self, value):
+                print(f"updating name of {cls.__name__}/{self.name} in cached prefix")
+                if self.name in cls:
+                    del cls[self.name]
+                actual_fset(self, value)
+                cls[self.name] = self
+
+            name_property = property(
+                name_property.fget,
+                updating_fset,
+                name_property.fdel,
+                name_property.__doc__,
+            )
+            setattr(cls, "name", name_property)
+
     def __contains__(cls, key):
         return f"{cls.asset_folder}/{key}" in Prefix()
 
@@ -191,3 +234,8 @@ class PrefixMeta(type):
         folder = f"{cls.asset_folder}/{key}"
         prefix = Prefix()
         prefix[folder] = value
+
+    def __delitem__(cls, key):
+        folder = f"{cls.asset_folder}/{key}"
+        prefix = Prefix()
+        del prefix[folder]

beat/backend/python/data.py

@@ -351,7 +351,7 @@ class CachedDataSource(DataSource):
                     )
                 self.dataformat = algo.result_dataformat()
             else:
-                self.dataformat = DataFormat(self.prefix, dataformat_name)
+                self.dataformat = DataFormat[dataformat_name]
 
             if not self.dataformat.valid:
                 raise RuntimeError(
@@ -614,7 +614,7 @@ class DatabaseOutputDataSource(DataSource):
         self.output_name = output_name
         self.pack = pack
 
-        self.dataformat = DataFormat(self.prefix, dataformat_name)
+        self.dataformat = DataFormat[dataformat_name]
 
         if not self.dataformat.valid:
             raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)
@@ -731,7 +731,7 @@ class RemoteDataSource(DataSource):
         self.input_name = input_name
         self.unpack = unpack
 
-        self.dataformat = DataFormat(prefix, dataformat_name)
+        self.dataformat = DataFormat[dataformat_name]
 
         if not self.dataformat.valid:
             raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)

beat/backend/python/database.py

@@ -60,6 +60,9 @@ from .exceptions import OutputError
 from .outputs import OutputList
 from .protocoltemplate import ProtocolTemplate
 
+DATABASE_TYPE = "database"
+DATABASE_FOLDER = "databases"
+
 # ----------------------------------------------------------
 
 
@@ -73,20 +76,21 @@ class Storage(utils.CodeStorage):
 
     """
 
-    asset_type = "database"
-    asset_folder = "databases"
+    asset_type = DATABASE_TYPE
+    asset_folder = DATABASE_FOLDER
 
-    def __init__(self, name):
+    def __init__(self, name, prefix=None):
 
         if name.count("/") != 1:
             raise RuntimeError("invalid database name: `%s'" % name)
 
         self.name, self.version = name.split("/")
         self.fullname = name
+        if prefix is None:
+            prefix = config.get_config()["prefix"]
+        self.prefix = prefix
 
-        path = os.path.join(
-            config.get_config()["prefix"], self.asset_folder, name + ".json"
-        )
+        path = os.path.join(prefix, self.asset_folder, name + ".json")
         path = path[:-5]
         # views are coded in Python
         super(Storage, self).__init__(path, "python")
@@ -217,7 +221,7 @@ class Runner(object):
 # ----------------------------------------------------------
 
 
-class Database(object):
+class Database(metaclass=config.PrefixMeta):
     """Databases define the start point of the dataflow in an experiment.
 
 
@@ -234,12 +238,16 @@ class Database(object):
 
     """
 
+    asset_type = DATABASE_TYPE
+    asset_folder = DATABASE_FOLDER
+
     def _init(self):
         self._name = None
         self.dataformats = {}  # preloaded dataformats
         self.storage = None
         self.errors = []
         self.data = None
+        return self
 
     def __init__(self, name):
 
@@ -256,8 +264,7 @@ class Database(object):
         schema_version=2,
         root_folder="/foo/bar",
     ):
-        self = cls.__new__(cls)
-        self._init()
+        self = cls.__new__(cls)._init()
 
         if not name:
             raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
@@ -275,7 +282,7 @@ class Database(object):
         for i, proto in enumerate(protocols):
             protocols[i]["template"] = protocoltemplate_name(proto["template"])
 
-        data = dict(protocols=protocols)
+        data = dict(protocols=protocols, root_folder=root_folder)
         if description is not None:
             data["description"] = description
         if schema_version is not None:
@@ -283,12 +290,11 @@ class Database(object):
 
         self.data = data
 
-        self.storage = Storage(name)
-        # save the code into storage
+        self.storage = Storage(self.name)
+        self.code_path = self.storage.code.path = code_path
         with open(code_path, "rt") as f:
-            self.storage.code.save(f.read())
-        self.code_path = self.storage.code.path
-        self.code = self.storage.code.load()
+            self.code = self.storage.code.contents = f.read()
+        self.storage.json.contents = str(self)
 
         self._load_v2()
 
@@ -496,7 +502,8 @@ class Database(object):
             protocol = self.protocol(protocol_name)
             template_name = protocol["template"]
             protocol_template = ProtocolTemplate[template_name]
-            view_definition = protocol_template.set(set_name)
+            # copy the set so we don't modify the original set
+            view_definition = dict(protocol_template.set(set_name))
             view_definition["view"] = protocol["views"][set_name]["view"]
             parameters = protocol["views"][set_name].get("parameters")
             if parameters is not None:
@@ -629,11 +636,6 @@ class Database(object):
         if not self.valid:
             raise RuntimeError("database is not valid")
 
-        if prefix == config.get_config()["prefix"]:
-            raise RuntimeError(
-                "Cannot export database to the same prefix (" "%s)" % prefix
-            )
-
         for k in self.dataformats.values():
             k.export(prefix)
 
@@ -642,7 +644,7 @@ class Database(object):
                 protocol_template = ProtocolTemplate[protocol["template"]]
                 protocol_template.export(prefix)
 
-        self.write(Storage(self.name))
+        self.write(Storage(self.name, prefix=prefix))
 
 
 # ----------------------------------------------------------

beat/backend/python/dataformat.py

@@ -54,8 +54,8 @@ from . import config
 from . import utils
 from .baseformat import baseformat
 
-DATA_FORMAT_TYPE = "dataformat"
-DATA_FORMAT_FOLDER = "dataformats"
+DATAFORMAT_TYPE = "dataformat"
+DATAFORMAT_FOLDER = "dataformats"
 
 # ----------------------------------------------------------
 
@@ -70,25 +70,27 @@ class Storage(utils.Storage):
 
     """
 
-    asset_type = DATA_FORMAT_TYPE
-    asset_folder = DATA_FORMAT_FOLDER
+    asset_type = DATAFORMAT_TYPE
+    asset_folder = DATAFORMAT_FOLDER
 
-    def __init__(self, name):
+    def __init__(self, name, prefix=None):
 
         if name.count("/") != 2:
             raise RuntimeError("invalid dataformat name: `%s'" % name)
 
         self.username, self.name, self.version = name.split("/")
         self.fullname = name
-        prefix = config.get_config()["prefix"]
+        if prefix is None:
+            prefix = config.get_config()["prefix"]
+        self.prefix = prefix
 
         path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
         path = path[:-5]
-        super(Storage, self).__init__(path)
+        super().__init__(path)
 
     def hash(self):
         """The 64-character hash of the database declaration JSON"""
-        return super(Storage, self).hash("#description")
+        return super().hash("#description")
 
 
 # ----------------------------------------------------------
@@ -141,8 +143,8 @@ class DataFormat(metaclass=config.PrefixMeta):
 
     """
 
-    asset_type = DATA_FORMAT_TYPE
-    asset_folder = DATA_FORMAT_FOLDER
+    asset_type = DATAFORMAT_TYPE
+    asset_folder = DATAFORMAT_FOLDER
 
     def _init(self):
         self._name = None
@@ -153,24 +155,23 @@ class DataFormat(metaclass=config.PrefixMeta):
         self.resolved = None
         self.referenced = {}
         self.parent = None
+        return self
 
     def __init__(self, data, parent=None):
 
         self._init()
         self.parent = parent
         self._load(data)
-        # cache in prefix
-        DataFormat[self.name] = self
 
     def _load(self, data):
         """Loads the dataformat"""
 
         if isinstance(data, dict):
-            self._name = "analysis:result"
+            self.name = "analysis:result"
             self.data = data
         else:
-            self._name = data
-            self.storage = Storage(data)
+            self.name = data
+            self.storage = Storage(self.name)
             json_path = self.storage.json.path
             if not self.storage.exists():
                 self.errors.append(
@@ -264,8 +265,7 @@ class DataFormat(metaclass=config.PrefixMeta):
         schema_version=None,
         parent=None,
     ):
-        self = cls.__new__(cls)
-        self._init()
+        self = cls.__new__(cls)._init()
 
         def str_or_dtype_or_type(v):
             # if it is a dict
@@ -301,21 +301,16 @@ class DataFormat(metaclass=config.PrefixMeta):
         if not name:
             raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
 
-        if name != "analysis:result" and "/" not in name:
-            name = f"{config.get_config()['user']}/{name}/1"
-
-        self._name = name
+        self.name = name
 
         self.parent = parent
 
-        if name != "analysis:result":
-            self.storage = Storage(name)
+        if self.name != "analysis:result":
+            self.storage = Storage(self.name)
+            self.storage.json.contents = str(self)
 
         self._resolve()
 
-        # cache in prefix
-        DataFormat[self.name] = self
-
         return self
 
     @property
@@ -330,8 +325,11 @@ class DataFormat(metaclass=config.PrefixMeta):
 
     @name.setter
     def name(self, value):
+        if value != "analysis:result" and "/" not in value:
+            value = f"{config.get_config()['user']}/{value}/1"
         self._name = value
-        self.storage = Storage(value)
+        if value != "analysis:result":
+            self.storage = Storage(value)
 
     @property
     def schema_version(self):
@@ -578,12 +576,7 @@ class DataFormat(metaclass=config.PrefixMeta):
         if not self.valid:
             raise RuntimeError("dataformat is not valid:\n{}".format(self.errors))
 
-        if prefix == prefix:
-            raise RuntimeError(
-                "Cannot export dataformat to the same prefix (" "%s)" % prefix
-            )
-
         for k in self.referenced.values():
             k.export(prefix)
 
-        self.write(Storage(prefix, self.name))
+        self.write(Storage(self.name, prefix=prefix))

beat/backend/python/execution/algorithm.py

@@ -181,6 +181,61 @@ class AlgorithmExecutor(object):
             self.loop_channel = LoopChannel(self.loop_socket)
             self.loop_channel.setup(self.algorithm, self.prefix)
 
+    @classmethod
+    def new(
+        cls,
+        data,
+        socket,
+        db_socket=None,
+        loop_socket=None,
+        databases=None,
+        cache_root="/cache",
+    ):
+        self = cls.__new__(cls)
+        self.data = data
+        self.socket = socket
+        self.db_socket = db_socket
+        self.loop_socket = loop_socket
+        self.loop_channel = None
+        self._runner = None
+        self.prefix = None
+
+        # Load the algorithm
+        self.algorithm = self.data["algorithm"]
+
+        if db_socket:
+            db_access_mode = AccessMode.REMOTE
+        else:
+            db_access_mode = AccessMode.LOCAL
+
+        (self.input_list, self.data_loaders) = create_inputs_from_configuration(
+            self.data,
+            self.algorithm,
+            prefix=self.prefix,
+            cache_root=cache_root,
+            cache_access=AccessMode.LOCAL,
+            db_access=db_access_mode,
+            socket=self.db_socket,
+            databases=databases,
+        )
+
+        # Loads algorithm outputs
+        (self.output_list, _) = create_outputs_from_configuration(
+            self.data,
+            self.algorithm,
+            prefix=self.prefix,
+            cache_root=cache_root,
+            input_list=self.input_list,
+            data_loaders=self.data_loaders,
+            loop_socket=self.loop_socket,
+        )
+
+        if self.loop_socket:
+            self.loop_channel = LoopChannel(self.loop_socket)
+            self.loop_channel.setup(self.algorithm, self.prefix)
+
+        return self
+
     @property
     def runner(self):
         """Returns the algorithm runner

beat/backend/python/hash.py

@@ -48,7 +48,6 @@ import hashlib
 import os
 
 import simplejson
-import six
 
 # ----------------------------------------------------------
 
@@ -57,11 +56,8 @@ def _sha256(s):
     """A python2/3 shortcut for :py:func:`haslib.sha256.hexdigest` to will
     ensure that the given string is unicode before going further.
     """
-    if isinstance(s, six.string_types):
-        try:
-            s = six.u(s).encode("utf-8")
-        except Exception:
-            s = s.encode("utf-8")
+    if isinstance(s, str):
+        s = s.encode("utf-8")
     return hashlib.sha256(s).hexdigest()
 
 

beat/backend/python/library.py

@@ -46,9 +46,13 @@ import os
 
 import simplejson as json
 
+from . import config
 from . import loader
 from . import utils
 
+LIBRARY_TYPE = "library"
+LIBRARY_FOLDER = "libraries"
+
 # ----------------------------------------------------------
 
 
@@ -57,29 +61,26 @@ class Storage(utils.CodeStorage):
 
     Parameters:
 
-      prefix (str): Establishes the prefix of
-        your installation.
-
       name (str): The name of the library object in the format
         ``<user>/<name>/<version>``.
 
     """
 
-    asset_type = "library"
-    asset_folder = "libraries"
+    asset_type = LIBRARY_TYPE
+    asset_folder = LIBRARY_FOLDER
 
-    def __init__(self, prefix, name, language=None):
+    def __init__(self, name, language=None, prefix=None):
 
         if name.count("/") != 2:
             raise RuntimeError("invalid library name: `%s'" % name)
 
         self.username, self.name, self.version = name.split("/")
         self.fullname = name
+        if prefix is None:
+            prefix = config.get_config()["prefix"]
         self.prefix = prefix
 
-        path = utils.hashed_or_simple(
-            self.prefix, self.asset_folder, name, suffix=".json"
-        )
+        path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
         path = path[:-5]
         super(Storage, self).__init__(path, language)
 
@@ -87,7 +88,7 @@ class Storage(utils.CodeStorage):
 # ----------------------------------------------------------
 
 
-class Library(object):
+class Library(metaclass=config.PrefixMeta):
     """Librarys represent independent algorithm components within the platform.
 
     This class can only parse the meta-parameters of the library. The actual
@@ -97,15 +98,8 @@ class Library(object):
 
     Parameters:
 
-      prefix (str): Establishes the prefix of your installation.
-
       name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
 
-      library_cache (:py:class:`dict`, Optional): A dictionary mapping library
-        names to loaded libraries. This parameter is optional and, if passed,
-        may greatly speed-up library loading times as libraries that are
-        already loaded may be re-used.
-
 
     Attributes:
 
@@ -136,28 +130,58 @@ class Library(object):
 
     """
 
-    def __init__(self, prefix, name, library_cache=None):
+    asset_type = LIBRARY_TYPE
+    asset_folder = LIBRARY_FOLDER
 
+    def _init(self):
         self._name = None
         self.storage = None
-        self.prefix = prefix
         self.errors = []
         self.libraries = {}
+        return self
+
+    def __init__(self, name):
+        self._init()
+        self._load(name)
+
+    @classmethod
+    def new(
+        cls, code_path, name, description=None, language="python", uses=None,
+    ):
+        self = cls.__new__(cls)._init()
+
+        def lib_name(v):
+            if hasattr(v, "name"):
+                v = v.name
+            return v
 
-        library_cache = library_cache if library_cache is not None else {}
+        uses = {k: lib_name(v) for k, v in (uses or {}).items()} or None
+        data = dict(language=language, description=description, uses=uses)
+        data = {k: v for k, v in data.items() if v is not None}
 
-        try:
-            self._load(name, library_cache)
-        finally:
-            if self._name is not None:  # registers it into the cache, even if failed
-                library_cache[self._name] = self
+        self.data = data
 
-    def _load(self, data, library_cache):
+        if not name:
+            raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
+
+        self.name = name
+
+        self.storage = Storage(self.name)
+        self.code_path = self.storage.code.path = code_path
+        with open(code_path, "rt") as f:
+            self.code = self.storage.code.contents = f.read()
+        self.storage.json.contents = str(self)
+
+        self._resolve()
+
+        return self
+
+    def _load(self, data):
         """Loads the library"""
 
-        self._name = data
+        self.name = data
 
-        self.storage = Storage(self.prefix, data)
+        self.storage = Storage(self.name)
         json_path = self.storage.json.path
         if not self.storage.exists():
             self.errors.append("Library declaration file not found: %s" % json_path)
@@ -178,11 +202,12 @@ class Library(object):
         # if no errors so far, make sense out of the library data
         self.data.setdefault("uses", {})
 
+        self._resolve()
+
+    def _resolve(self):
         if self.uses is not None:
             for name, value in self.uses.items():
-                self.libraries[value] = Library(self.prefix, value, library_cache)
-
-        self.libraries[self._name] = self
+                self.libraries[value] = Library[value]
 
     def uses_dict(self):
         """Returns the usage dictionary for all dependent modules"""
@@ -233,8 +258,11 @@ class Library(object):
         if self.data["language"] == "unknown":
             raise RuntimeError("library has no programming language set")
 
+        if "/" not in value:
+            value = f"{config.get_config()['user']}/{value}/1"
+
         self._name = value
-        self.storage = Storage(self.prefix, value, self.data["language"])
+        self.storage = Storage(value, self.data["language"])
 
     @property
     def schema_version(self):
@@ -252,7 +280,6 @@ class Library(object):
         if self.storage:
             self.storage.language = value
         self.data["language"] = value
-        self._check_language_consistence()
 
     @property
     def valid(self):
@@ -369,7 +396,7 @@ class Library(object):
 
         Raises:
 
-          RuntimeError: If prefix and self.prefix point to the same directory.
+          RuntimeError: If prefix and prefix point to the same directory.
 
         """
 
@@ -379,12 +406,7 @@ class Library(object):
         if not self.valid:
             raise RuntimeError("library is not valid")
 
-        if prefix == self.prefix:
-            raise RuntimeError(
-                "Cannot export library to the same prefix (" "%s)" % (prefix)
-            )
-
         for k in self.libraries.values():
             k.export(prefix)
 
-        self.write(Storage(prefix, self.name))
+        self.write(Storage(self.name, prefix=prefix))

beat/backend/python/protocoltemplate.py

@@ -48,8 +48,8 @@ from . import config
 from . import utils
 from .dataformat import DataFormat
 
-PROTOCOL_TEMPLATE_TYPE = "protocoltemplate"
-PROTOCOL_TEMPLATE_FOLDER = "protocoltemplates"
+PROTOCOLTEMPLATE_TYPE = "protocoltemplate"
+PROTOCOLTEMPLATE_FOLDER = "protocoltemplates"
 
 
 # ----------------------------------------------------------
@@ -63,20 +63,21 @@ class Storage(utils.Storage):
 
     """
 
-    asset_type = PROTOCOL_TEMPLATE_TYPE
-    asset_folder = PROTOCOL_TEMPLATE_FOLDER
+    asset_type = PROTOCOLTEMPLATE_TYPE
+    asset_folder = PROTOCOLTEMPLATE_FOLDER
 
-    def __init__(self, name):
+    def __init__(self, name, prefix=None):
 
         if name.count("/") != 1:
             raise RuntimeError("invalid protocol template name: `%s'" % name)
 
         self.name, self.version = name.split("/")
         self