diff --git a/beat/backend/python/algorithm.py b/beat/backend/python/algorithm.py
index 08b8128825584c265a9a94c9a266049c67238e04..27573eb2466fad96a56f65eddee5d9c0fbaca626 100644
--- a/beat/backend/python/algorithm.py
+++ b/beat/backend/python/algorithm.py
@@ -50,13 +50,17 @@ import numpy
import simplejson as json
import six
+from . import config
from . import dataformat
-from . import library
from . import loader
from . import utils
+from .library import Library
logger = logging.getLogger(__name__)
+ALGORITHM_TYPE = "algorithm"
+ALGORITHM_FOLDER = "algorithms"
+
# ----------------------------------------------------------
@@ -66,28 +70,26 @@ class Storage(utils.CodeStorage):
Parameters:
- prefix (str): Establishes the prefix of your installation.
-
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
"""
- asset_type = "algorithm"
- asset_folder = "algorithms"
+ asset_type = ALGORITHM_TYPE
+ asset_folder = ALGORITHM_FOLDER
- def __init__(self, prefix, name, language=None):
+ def __init__(self, name, language=None, prefix=None):
if name.count("/") != 2:
raise RuntimeError("invalid algorithm name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
+ if prefix is None:
+ prefix = config.get_config()["prefix"]
self.prefix = prefix
- path = utils.hashed_or_simple(
- self.prefix, self.asset_folder, name, suffix=".json"
- )
+ path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
super(Storage, self).__init__(path, language)
@@ -375,7 +377,16 @@ class Runner(object):
# ----------------------------------------------------------
-class Algorithm(object):
+def _to_name(v, convert_basic_types=False):
+ if hasattr(v, "name"):
+ v = v.name
+ if convert_basic_types:
+ if not isinstance(v, str):
+ v = numpy.dtype(v).name
+ return v
+
+
+class Algorithm(metaclass=config.PrefixMeta):
"""Algorithms represent runnable components within the platform.
This class can only parse the meta-parameters of the algorithm (i.e., input
@@ -387,20 +398,8 @@ class Algorithm(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
-
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
- dataformat_cache (:py:class:`dict`, Optional): A dictionary mapping
- dataformat names to loaded dataformats. This parameter is optional and,
- if passed, may greatly speed-up algorithm loading times as dataformats
- that are already loaded may be re-used.
-
- library_cache (:py:class:`dict`, Optional): A dictionary mapping library
- names to loaded libraries. This parameter is optional and, if passed,
- may greatly speed-up library loading times as libraries that are
- already loaded may be re-used.
-
Attributes:
@@ -449,6 +448,9 @@ class Algorithm(object):
"""
+ asset_type = ALGORITHM_TYPE
+ asset_folder = ALGORITHM_FOLDER
+
LEGACY = "legacy"
SEQUENTIAL = "sequential"
AUTONOMOUS = "autonomous"
@@ -459,27 +461,83 @@ class Algorithm(object):
dataformat_klass = dataformat.DataFormat
- def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
-
+ def _init(self):
self._name = None
self.storage = None
- self.prefix = prefix
self.dataformats = {}
self.libraries = {}
self.groups = []
self.errors = []
+ return self
+
+ def __init__(self, name):
+
+ self._init()
+ self._load(name)
+
+ @classmethod
+ def new(
+ cls,
+ code_path,
+ name,
+ groups,
+ parameters=None,
+ description=None,
+ type="autonomous",
+ splittable=False,
+ api_version=2,
+ language="python",
+ uses=None,
+ schema_version=2,
+ **kwargs,
+ ):
+ self = cls.__new__(cls)._init()
+
+ uses = {k: _to_name(v) for k, v in (uses or {}).items()} or None
+ # convert dataformats to their names in groups
+ for i, grp in enumerate(groups):
+ for key, value in grp.items():
+ if key not in ("inputs", "outputs"):
+ continue
+ for node_details in value.values():
+ node_details["type"] = _to_name(node_details["type"])
+
+ data = dict(
+ api_version=api_version,
+ description=description,
+ groups=groups,
+ language=language,
+ parameters=parameters,
+ schema_version=schema_version,
+ splittable=splittable,
+ type=type,
+ uses=uses,
+ )
+ data = {k: v for k, v in data.items() if v is not None}
+
+ self.data = data
+
+ if not name:
+ raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
- dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
- library_cache = library_cache if library_cache is not None else {}
+ self.name = name
- self._load(name, dataformat_cache, library_cache)
+ self.storage = Storage(self.name)
+ self.code_path = self.storage.code.path = code_path
+ with open(code_path, "rt") as f:
+ self.code = self.storage.code.contents = f.read()
+ self.storage.json.contents = str(self)
- def _load(self, data, dataformat_cache, library_cache):
+ self._resolve()
+
+ return self
+
+ def _load(self, name):
"""Loads the algorithm"""
- self._name = data
+ self.name = name
- self.storage = Storage(self.prefix, data)
+ self.storage = Storage(self.name)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append("Algorithm declaration file not found: %s" % json_path)
@@ -497,7 +555,9 @@ class Algorithm(object):
self.code_path = self.storage.code.path
self.code = self.storage.code.load()
+ self._resolve()
+ def _resolve(self):
self.groups = self.data["groups"]
# create maps for easy access to data
@@ -511,46 +571,46 @@ class Algorithm(object):
for k, v in g.get("outputs", {}).items()
]
)
+ # find the group name of outputs
+ self.output_group = None
+ for g in self.groups:
+ for k, v in g.get("outputs", {}).items():
+ self.output_group = g
+ break
+ break
+
self.loop_map = dict(
[(k, v["type"]) for g in self.groups for k, v in g.get("loop", {}).items()]
)
- self._load_dataformats(dataformat_cache)
+ self._load_dataformats()
self._convert_parameter_types()
- self._load_libraries(library_cache)
+ self._load_libraries()
- def _update_dataformat_cache(self, type_name, dataformat_cache):
+ def _update_dataformat_cache(self, type_name):
"""Update the data format cache based on the type name"""
if type_name not in self.dataformats:
- if dataformat_cache and type_name in dataformat_cache: # reuse
- thisformat = dataformat_cache[type_name]
- else: # load it
- thisformat = self.dataformat_klass(self.prefix, type_name)
- if dataformat_cache is not None: # update it
- dataformat_cache[type_name] = thisformat
-
+ thisformat = self.dataformat_klass[type_name]
self.dataformats[type_name] = thisformat
return self.dataformats[type_name]
- def _update_dataformat_cache_for_group(self, group, dataformat_cache):
+ def _update_dataformat_cache_for_group(self, group):
for _, entry in group.items():
- self._update_dataformat_cache(entry["type"], dataformat_cache)
+ self._update_dataformat_cache(entry["type"])
- def _load_dataformats(self, dataformat_cache):
+ def _load_dataformats(self):
"""Makes sure we can load all requested formats
"""
for group in self.groups:
- self._update_dataformat_cache_for_group(group["inputs"], dataformat_cache)
+ self._update_dataformat_cache_for_group(group["inputs"])
if "outputs" in group:
- self._update_dataformat_cache_for_group(
- group["outputs"], dataformat_cache
- )
+ self._update_dataformat_cache_for_group(group["outputs"])
if "loop" in group:
- self._update_dataformat_cache_for_group(group["loop"], dataformat_cache)
+ self._update_dataformat_cache_for_group(group["loop"])
if self.results:
@@ -559,7 +619,7 @@ class Algorithm(object):
# results can only contain base types and plots therefore, only
# process plots
if result_type.find("/") != -1:
- self._update_dataformat_cache(result_type, dataformat_cache)
+ self._update_dataformat_cache(result_type)
def _convert_parameter_types(self):
"""Converts types to numpy equivalents, checks defaults, ranges and choices
@@ -639,7 +699,7 @@ class Algorithm(object):
)
)
- def _load_libraries(self, library_cache):
+ def _load_libraries(self):
# all used libraries must be loadable; cannot use self as a library
@@ -647,9 +707,7 @@ class Algorithm(object):
for name, value in self.uses.items():
- self.libraries[value] = library_cache.setdefault(
- value, library.Library(self.prefix, value, library_cache)
- )
+ self.libraries[value] = Library[value]
@property
def name(self):
@@ -668,8 +726,11 @@ class Algorithm(object):
if self.data["language"] == "unknown":
raise RuntimeError("algorithm has no programming language set")
+ if "/" not in value:
+ value = f"{config.get_config()['user']}/{value}/1"
+
self._name = value
- self.storage = Storage(self.prefix, value, self.data["language"])
+ self.storage = Storage(value, self.data["language"])
@property
def schema_version(self):
@@ -898,7 +959,7 @@ class Algorithm(object):
)
format = dataformat.DataFormat(
- self.prefix, dict([(k, v["type"]) for k, v in self.results.items()])
+ dict([(k, v["type"]) for k, v in self.results.items()])
)
format.name = "analysis:" + self.name
@@ -1034,7 +1095,7 @@ class Algorithm(object):
Raises:
- RuntimeError: If prefix and self.prefix point to the same directory.
+ RuntimeError: If prefix and prefix point to the same directory.
"""
@@ -1044,19 +1105,53 @@ class Algorithm(object):
if not self.valid:
raise RuntimeError("algorithm is not valid:\n%s" % "\n".join(self.errors))
- if prefix == self.prefix:
- raise RuntimeError(
- "Cannot export algorithm to the same prefix (" "%s)" % prefix
- )
-
for k in self.libraries.values():
k.export(prefix)
for k in self.dataformats.values():
k.export(prefix)
- self.write(Storage(prefix, self.name, self.language))
+ self.write(Storage(self.name, self.language, prefix=prefix))
class Analyzer(Algorithm):
"""docstring for Analyzer"""
+
+ @classmethod
+ def new(
+ cls,
+ code_path,
+ name,
+ groups,
+ results,
+ description=None,
+ type="autonomous",
+ api_version=2,
+ language="python",
+ uses=None,
+ schema_version=2,
+ **kwargs,
+ ):
+
+ self = super().new(
+ code_path=code_path,
+ name=name,
+ groups=groups,
+ description=description,
+ type=type,
+ api_version=api_version,
+ language=language,
+ uses=uses,
+ schema_version=schema_version,
+ # no parameters or splittable for analyzers
+ parameters=None,
+ splittable=None,
+ **kwargs,
+ )
+ # convert dataformats to their names in results
+ for name, details in results.items():
+ details["type"] = _to_name(details["type"], convert_basic_types=True)
+
+ self.data["results"] = results
+ self._resolve()
+ return self
diff --git a/beat/backend/python/config.py b/beat/backend/python/config.py
index 0a04f6c7c2110cad9edf0180b97579b2b526e0fa..3ef4df2438472d5f8b2a99eb675c197565b70639 100644
--- a/beat/backend/python/config.py
+++ b/beat/backend/python/config.py
@@ -99,7 +99,7 @@ def set_config(**kwargs):
"""
supported_keys = set(DEFAULTS.keys())
set_keys = set(kwargs.keys())
- if set_keys not in supported_keys:
+ if not set_keys.issubset(supported_keys):
raise ValueError(
f"Only {supported_keys} are valid configurations. "
f"Got these extra values: {set_keys - supported_keys}"
@@ -137,14 +137,14 @@ def config_context(**new_config):
"""
old_config = get_config().copy()
# also backup prefix
- old_prefix = Prefix().copy()
+ prefix = Prefix()
+ old_prefix = prefix.copy()
set_config(**new_config)
try:
yield
finally:
set_config(**old_config)
- prefix = Prefix()
prefix.clear()
prefix.update(old_prefix)
@@ -154,7 +154,6 @@ def config_context(**new_config):
class Singleton(type):
"""A Singleton metaclass
- The singleton class calls the __init__ method each time the instance is requested.
From: https://stackoverflow.com/a/6798042/1286165
"""
@@ -167,11 +166,55 @@ class Singleton(type):
class Prefix(dict, metaclass=Singleton):
- def __init__(self, path=None, *args, **kwargs):
- super().__init__(*args, **kwargs)
+ pass
class PrefixMeta(type):
+ # cache instances when __init__ is called
+ def __call__(cls, *args, **kwargs):
+ instance = super().__call__(*args, **kwargs)
+
+ folder = f"{cls.asset_folder}/{instance.name}"
+ prefix = Prefix()
+ prefix[folder] = instance
+ print(f"caching {cls.__name__}/{instance.name} in __init__ calls")
+
+ return instance
+
+ def __init__(cls, name, bases, clsdict):
+ # cache instances when new is called
+ if "new" in clsdict:
+
+ def caching_new(*args, **kwargs):
+ instance = clsdict["new"].__func__(cls, *args, **kwargs)
+ folder = f"{cls.asset_folder}/{instance.name}"
+ prefix = Prefix()
+ prefix[folder] = instance
+ print(f"caching {cls.__name__}/{instance.name} in new calls")
+ return instance
+
+ setattr(cls, "new", caching_new)
+
+ # change the key of instance when its name is changed
+ if "name" in clsdict:
+ name_property = clsdict["name"]
+ actual_fset = name_property.fset
+
+ def updating_fset(self, value):
+ print(f"updating name of {cls.__name__}/{self.name} in cached prefix")
+ if self.name in cls:
+ del cls[self.name]
+ actual_fset(self, value)
+ cls[self.name] = self
+
+ name_property = property(
+ name_property.fget,
+ updating_fset,
+ name_property.fdel,
+ name_property.__doc__,
+ )
+ setattr(cls, "name", name_property)
+
def __contains__(cls, key):
return f"{cls.asset_folder}/{key}" in Prefix()
@@ -191,3 +234,8 @@ class PrefixMeta(type):
folder = f"{cls.asset_folder}/{key}"
prefix = Prefix()
prefix[folder] = value
+
+ def __delitem__(cls, key):
+ folder = f"{cls.asset_folder}/{key}"
+ prefix = Prefix()
+ del prefix[folder]
diff --git a/beat/backend/python/data.py b/beat/backend/python/data.py
index b38b8b1b668abafde6a8902c941f22344308a735..432358ed633f49282188b396846944fe6c1fc64e 100644
--- a/beat/backend/python/data.py
+++ b/beat/backend/python/data.py
@@ -351,7 +351,7 @@ class CachedDataSource(DataSource):
)
self.dataformat = algo.result_dataformat()
else:
- self.dataformat = DataFormat(self.prefix, dataformat_name)
+ self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError(
@@ -614,7 +614,7 @@ class DatabaseOutputDataSource(DataSource):
self.output_name = output_name
self.pack = pack
- self.dataformat = DataFormat(self.prefix, dataformat_name)
+ self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)
@@ -731,7 +731,7 @@ class RemoteDataSource(DataSource):
self.input_name = input_name
self.unpack = unpack
- self.dataformat = DataFormat(prefix, dataformat_name)
+ self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)
diff --git a/beat/backend/python/database.py b/beat/backend/python/database.py
index 2383c0602c6cea3e9884770aae758a341e966571..bf8c784ffd80e59c7d6e357afd8edfa0c41ff90a 100644
--- a/beat/backend/python/database.py
+++ b/beat/backend/python/database.py
@@ -60,6 +60,9 @@ from .exceptions import OutputError
from .outputs import OutputList
from .protocoltemplate import ProtocolTemplate
+DATABASE_TYPE = "database"
+DATABASE_FOLDER = "databases"
+
# ----------------------------------------------------------
@@ -73,20 +76,21 @@ class Storage(utils.CodeStorage):
"""
- asset_type = "database"
- asset_folder = "databases"
+ asset_type = DATABASE_TYPE
+ asset_folder = DATABASE_FOLDER
- def __init__(self, name):
+ def __init__(self, name, prefix=None):
if name.count("/") != 1:
raise RuntimeError("invalid database name: `%s'" % name)
self.name, self.version = name.split("/")
self.fullname = name
+ if prefix is None:
+ prefix = config.get_config()["prefix"]
+ self.prefix = prefix
- path = os.path.join(
- config.get_config()["prefix"], self.asset_folder, name + ".json"
- )
+ path = os.path.join(prefix, self.asset_folder, name + ".json")
path = path[:-5]
# views are coded in Python
super(Storage, self).__init__(path, "python")
@@ -217,7 +221,7 @@ class Runner(object):
# ----------------------------------------------------------
-class Database(object):
+class Database(metaclass=config.PrefixMeta):
"""Databases define the start point of the dataflow in an experiment.
@@ -234,12 +238,16 @@ class Database(object):
"""
+ asset_type = DATABASE_TYPE
+ asset_folder = DATABASE_FOLDER
+
def _init(self):
self._name = None
self.dataformats = {} # preloaded dataformats
self.storage = None
self.errors = []
self.data = None
+ return self
def __init__(self, name):
@@ -256,8 +264,7 @@ class Database(object):
schema_version=2,
root_folder="/foo/bar",
):
- self = cls.__new__(cls)
- self._init()
+ self = cls.__new__(cls)._init()
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
@@ -275,7 +282,7 @@ class Database(object):
for i, proto in enumerate(protocols):
protocols[i]["template"] = protocoltemplate_name(proto["template"])
- data = dict(protocols=protocols)
+ data = dict(protocols=protocols, root_folder=root_folder)
if description is not None:
data["description"] = description
if schema_version is not None:
@@ -283,12 +290,11 @@ class Database(object):
self.data = data
- self.storage = Storage(name)
- # save the code into storage
+ self.storage = Storage(self.name)
+ self.code_path = self.storage.code.path = code_path
with open(code_path, "rt") as f:
- self.storage.code.save(f.read())
- self.code_path = self.storage.code.path
- self.code = self.storage.code.load()
+ self.code = self.storage.code.contents = f.read()
+ self.storage.json.contents = str(self)
self._load_v2()
@@ -496,7 +502,8 @@ class Database(object):
protocol = self.protocol(protocol_name)
template_name = protocol["template"]
protocol_template = ProtocolTemplate[template_name]
- view_definition = protocol_template.set(set_name)
+ # copy the set so we don't modify the original set
+ view_definition = dict(protocol_template.set(set_name))
view_definition["view"] = protocol["views"][set_name]["view"]
parameters = protocol["views"][set_name].get("parameters")
if parameters is not None:
@@ -629,11 +636,6 @@ class Database(object):
if not self.valid:
raise RuntimeError("database is not valid")
- if prefix == config.get_config()["prefix"]:
- raise RuntimeError(
- "Cannot export database to the same prefix (" "%s)" % prefix
- )
-
for k in self.dataformats.values():
k.export(prefix)
@@ -642,7 +644,7 @@ class Database(object):
protocol_template = ProtocolTemplate[protocol["template"]]
protocol_template.export(prefix)
- self.write(Storage(self.name))
+ self.write(Storage(self.name, prefix=prefix))
# ----------------------------------------------------------
diff --git a/beat/backend/python/dataformat.py b/beat/backend/python/dataformat.py
index f5774ae6c37ad7a10a3ba83b8c63aa588479513c..a8ad589089742778c7fadb60d0b9e7ed0caa0ac1 100644
--- a/beat/backend/python/dataformat.py
+++ b/beat/backend/python/dataformat.py
@@ -54,8 +54,8 @@ from . import config
from . import utils
from .baseformat import baseformat
-DATA_FORMAT_TYPE = "dataformat"
-DATA_FORMAT_FOLDER = "dataformats"
+DATAFORMAT_TYPE = "dataformat"
+DATAFORMAT_FOLDER = "dataformats"
# ----------------------------------------------------------
@@ -70,25 +70,27 @@ class Storage(utils.Storage):
"""
- asset_type = DATA_FORMAT_TYPE
- asset_folder = DATA_FORMAT_FOLDER
+ asset_type = DATAFORMAT_TYPE
+ asset_folder = DATAFORMAT_FOLDER
- def __init__(self, name):
+ def __init__(self, name, prefix=None):
if name.count("/") != 2:
raise RuntimeError("invalid dataformat name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
- prefix = config.get_config()["prefix"]
+ if prefix is None:
+ prefix = config.get_config()["prefix"]
+ self.prefix = prefix
path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
- super(Storage, self).__init__(path)
+ super().__init__(path)
def hash(self):
"""The 64-character hash of the database declaration JSON"""
- return super(Storage, self).hash("#description")
+ return super().hash("#description")
# ----------------------------------------------------------
@@ -141,8 +143,8 @@ class DataFormat(metaclass=config.PrefixMeta):
"""
- asset_type = DATA_FORMAT_TYPE
- asset_folder = DATA_FORMAT_FOLDER
+ asset_type = DATAFORMAT_TYPE
+ asset_folder = DATAFORMAT_FOLDER
def _init(self):
self._name = None
@@ -153,24 +155,23 @@ class DataFormat(metaclass=config.PrefixMeta):
self.resolved = None
self.referenced = {}
self.parent = None
+ return self
def __init__(self, data, parent=None):
self._init()
self.parent = parent
self._load(data)
- # cache in prefix
- DataFormat[self.name] = self
def _load(self, data):
"""Loads the dataformat"""
if isinstance(data, dict):
- self._name = "analysis:result"
+ self.name = "analysis:result"
self.data = data
else:
- self._name = data
- self.storage = Storage(data)
+ self.name = data
+ self.storage = Storage(self.name)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append(
@@ -264,8 +265,7 @@ class DataFormat(metaclass=config.PrefixMeta):
schema_version=None,
parent=None,
):
- self = cls.__new__(cls)
- self._init()
+ self = cls.__new__(cls)._init()
def str_or_dtype_or_type(v):
# if it is a dict
@@ -301,21 +301,16 @@ class DataFormat(metaclass=config.PrefixMeta):
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
- if name != "analysis:result" and "/" not in name:
- name = f"{config.get_config()['user']}/{name}/1"
-
- self._name = name
+ self.name = name
self.parent = parent
- if name != "analysis:result":
- self.storage = Storage(name)
+ if self.name != "analysis:result":
+ self.storage = Storage(self.name)
+ self.storage.json.contents = str(self)
self._resolve()
- # cache in prefix
- DataFormat[self.name] = self
-
return self
@property
@@ -330,8 +325,11 @@ class DataFormat(metaclass=config.PrefixMeta):
@name.setter
def name(self, value):
+ if value != "analysis:result" and "/" not in value:
+ value = f"{config.get_config()['user']}/{value}/1"
self._name = value
- self.storage = Storage(value)
+ if value != "analysis:result":
+ self.storage = Storage(value)
@property
def schema_version(self):
@@ -578,12 +576,7 @@ class DataFormat(metaclass=config.PrefixMeta):
if not self.valid:
raise RuntimeError("dataformat is not valid:\n{}".format(self.errors))
- if prefix == prefix:
- raise RuntimeError(
- "Cannot export dataformat to the same prefix (" "%s)" % prefix
- )
-
for k in self.referenced.values():
k.export(prefix)
- self.write(Storage(prefix, self.name))
+ self.write(Storage(self.name, prefix=prefix))
diff --git a/beat/backend/python/execution/algorithm.py b/beat/backend/python/execution/algorithm.py
index abda1faee489875756dfd08e92ce6d8d1a5ae55d..99a1fb37ec7ab5c8927ab8dc5d1478a7f8d5f34d 100644
--- a/beat/backend/python/execution/algorithm.py
+++ b/beat/backend/python/execution/algorithm.py
@@ -181,6 +181,61 @@ class AlgorithmExecutor(object):
self.loop_channel = LoopChannel(self.loop_socket)
self.loop_channel.setup(self.algorithm, self.prefix)
+ @classmethod
+ def new(
+ cls,
+ data,
+ socket,
+ db_socket=None,
+ loop_socket=None,
+ databases=None,
+ cache_root="/cache",
+ ):
+ self = cls.__new__(cls)
+ self.data = data
+ self.socket = socket
+ self.db_socket = db_socket
+ self.loop_socket = loop_socket
+ self.loop_channel = None
+ self._runner = None
+ self.prefix = None
+
+ # Load the algorithm
+ self.algorithm = self.data["algorithm"]
+
+ if db_socket:
+ db_access_mode = AccessMode.REMOTE
+ else:
+ db_access_mode = AccessMode.LOCAL
+
+ (self.input_list, self.data_loaders) = create_inputs_from_configuration(
+ self.data,
+ self.algorithm,
+ prefix=self.prefix,
+ cache_root=cache_root,
+ cache_access=AccessMode.LOCAL,
+ db_access=db_access_mode,
+ socket=self.db_socket,
+ databases=databases,
+ )
+
+ # Loads algorithm outputs
+ (self.output_list, _) = create_outputs_from_configuration(
+ self.data,
+ self.algorithm,
+ prefix=self.prefix,
+ cache_root=cache_root,
+ input_list=self.input_list,
+ data_loaders=self.data_loaders,
+ loop_socket=self.loop_socket,
+ )
+
+ if self.loop_socket:
+ self.loop_channel = LoopChannel(self.loop_socket)
+ self.loop_channel.setup(self.algorithm, self.prefix)
+
+ return self
+
@property
def runner(self):
"""Returns the algorithm runner
diff --git a/beat/backend/python/hash.py b/beat/backend/python/hash.py
index c6a483bccf997a8fcb40132c49562575bb009312..10c0d80f9fe1110bda86ee6b5100c2b40470c718 100644
--- a/beat/backend/python/hash.py
+++ b/beat/backend/python/hash.py
@@ -48,7 +48,6 @@ import hashlib
import os
import simplejson
-import six
# ----------------------------------------------------------
@@ -57,11 +56,8 @@ def _sha256(s):
"""A python2/3 shortcut for :py:func:`haslib.sha256.hexdigest` to will
ensure that the given string is unicode before going further.
"""
- if isinstance(s, six.string_types):
- try:
- s = six.u(s).encode("utf-8")
- except Exception:
- s = s.encode("utf-8")
+ if isinstance(s, str):
+ s = s.encode("utf-8")
return hashlib.sha256(s).hexdigest()
diff --git a/beat/backend/python/library.py b/beat/backend/python/library.py
index 22bd721bc20215f061a12d465bc2a4da31de3785..5f50b863969b125c2326d4c3fd8718884aca4182 100644
--- a/beat/backend/python/library.py
+++ b/beat/backend/python/library.py
@@ -46,9 +46,13 @@ import os
import simplejson as json
+from . import config
from . import loader
from . import utils
+LIBRARY_TYPE = "library"
+LIBRARY_FOLDER = "libraries"
+
# ----------------------------------------------------------
@@ -57,29 +61,26 @@ class Storage(utils.CodeStorage):
Parameters:
- prefix (str): Establishes the prefix of
- your installation.
-
name (str): The name of the library object in the format
``<user>/<name>/<version>``.
"""
- asset_type = "library"
- asset_folder = "libraries"
+ asset_type = LIBRARY_TYPE
+ asset_folder = LIBRARY_FOLDER
- def __init__(self, prefix, name, language=None):
+ def __init__(self, name, language=None, prefix=None):
if name.count("/") != 2:
raise RuntimeError("invalid library name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
+ if prefix is None:
+ prefix = config.get_config()["prefix"]
self.prefix = prefix
- path = utils.hashed_or_simple(
- self.prefix, self.asset_folder, name, suffix=".json"
- )
+ path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
super(Storage, self).__init__(path, language)
@@ -87,7 +88,7 @@ class Storage(utils.CodeStorage):
# ----------------------------------------------------------
-class Library(object):
+class Library(metaclass=config.PrefixMeta):
"""Librarys represent independent algorithm components within the platform.
This class can only parse the meta-parameters of the library. The actual
@@ -97,15 +98,8 @@ class Library(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
-
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
- library_cache (:py:class:`dict`, Optional): A dictionary mapping library
- names to loaded libraries. This parameter is optional and, if passed,
- may greatly speed-up library loading times as libraries that are
- already loaded may be re-used.
-
Attributes:
@@ -136,28 +130,58 @@ class Library(object):
"""
- def __init__(self, prefix, name, library_cache=None):
+ asset_type = LIBRARY_TYPE
+ asset_folder = LIBRARY_FOLDER
+ def _init(self):
self._name = None
self.storage = None
- self.prefix = prefix
self.errors = []
self.libraries = {}
+ return self
+
+ def __init__(self, name):
+ self._init()
+ self._load(name)
+
+ @classmethod
+ def new(
+ cls, code_path, name, description=None, language="python", uses=None,
+ ):
+ self = cls.__new__(cls)._init()
+
+ def lib_name(v):
+ if hasattr(v, "name"):
+ v = v.name
+ return v
- library_cache = library_cache if library_cache is not None else {}
+ uses = {k: lib_name(v) for k, v in (uses or {}).items()} or None
+ data = dict(language=language, description=description, uses=uses)
+ data = {k: v for k, v in data.items() if v is not None}
- try:
- self._load(name, library_cache)
- finally:
- if self._name is not None: # registers it into the cache, even if failed
- library_cache[self._name] = self
+ self.data = data
- def _load(self, data, library_cache):
+ if not name:
+ raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
+
+ self.name = name
+
+ self.storage = Storage(self.name)
+ self.code_path = self.storage.code.path = code_path
+ with open(code_path, "rt") as f:
+ self.code = self.storage.code.contents = f.read()
+ self.storage.json.contents = str(self)
+
+ self._resolve()
+
+ return self
+
+ def _load(self, data):
"""Loads the library"""
- self._name = data
+ self.name = data
- self.storage = Storage(self.prefix, data)
+ self.storage = Storage(self.name)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append("Library declaration file not found: %s" % json_path)
@@ -178,11 +202,12 @@ class Library(object):
# if no errors so far, make sense out of the library data
self.data.setdefault("uses", {})
+ self._resolve()
+
+ def _resolve(self):
if self.uses is not None:
for name, value in self.uses.items():
- self.libraries[value] = Library(self.prefix, value, library_cache)
-
- self.libraries[self._name] = self
+ self.libraries[value] = Library[value]
def uses_dict(self):
"""Returns the usage dictionary for all dependent modules"""
@@ -233,8 +258,11 @@ class Library(object):
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
+ if "/" not in value:
+ value = f"{config.get_config()['user']}/{value}/1"
+
self._name = value
- self.storage = Storage(self.prefix, value, self.data["language"])
+ self.storage = Storage(value, self.data["language"])
@property
def schema_version(self):
@@ -252,7 +280,6 @@ class Library(object):
if self.storage:
self.storage.language = value
self.data["language"] = value
- self._check_language_consistence()
@property
def valid(self):
@@ -369,7 +396,7 @@ class Library(object):
Raises:
- RuntimeError: If prefix and self.prefix point to the same directory.
+ RuntimeError: If prefix and prefix point to the same directory.
"""
@@ -379,12 +406,7 @@ class Library(object):
if not self.valid:
raise RuntimeError("library is not valid")
- if prefix == self.prefix:
- raise RuntimeError(
- "Cannot export library to the same prefix (" "%s)" % (prefix)
- )
-
for k in self.libraries.values():
k.export(prefix)
- self.write(Storage(prefix, self.name))
+ self.write(Storage(self.name, prefix=prefix))
diff --git a/beat/backend/python/protocoltemplate.py b/beat/backend/python/protocoltemplate.py
index e14b1d9ecfd254c0562563b2f55435d8e8020e33..720bb111c0d3919d3770b1cef637c8d64680bb37 100644
--- a/beat/backend/python/protocoltemplate.py
+++ b/beat/backend/python/protocoltemplate.py
@@ -48,8 +48,8 @@ from . import config
from . import utils
from .dataformat import DataFormat
-PROTOCOL_TEMPLATE_TYPE = "protocoltemplate"
-PROTOCOL_TEMPLATE_FOLDER = "protocoltemplates"
+PROTOCOLTEMPLATE_TYPE = "protocoltemplate"
+PROTOCOLTEMPLATE_FOLDER = "protocoltemplates"
# ----------------------------------------------------------
@@ -63,20 +63,21 @@ class Storage(utils.Storage):
"""
- asset_type = PROTOCOL_TEMPLATE_TYPE
- asset_folder = PROTOCOL_TEMPLATE_FOLDER
+ asset_type = PROTOCOLTEMPLATE_TYPE
+ asset_folder = PROTOCOLTEMPLATE_FOLDER
- def __init__(self, name):
+ def __init__(self, name, prefix=None):
if name.count("/") != 1:
raise RuntimeError("invalid protocol template name: `%s'" % name)
self.name, self.version = name.split("/")
self.fullname = name
+ if prefix is None:
+ prefix = config.get_config()["prefix"]
+ self.prefix = prefix
- path = utils.hashed_or_simple(
- config.get_config()["prefix"], self.asset_folder, name, suffix=".json"
- )
+ path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
super(Storage, self).__init__(path)
@@ -101,8 +102,8 @@ class ProtocolTemplate(metaclass=config.PrefixMeta):
"""
- asset_type = PROTOCOL_TEMPLATE_TYPE
- asset_folder = PROTOCOL_TEMPLATE_FOLDER
+ asset_type = PROTOCOLTEMPLATE_TYPE
+ asset_folder = PROTOCOLTEMPLATE_FOLDER
def _init(self):
self._name = None
@@ -110,21 +111,18 @@ class ProtocolTemplate(metaclass=config.PrefixMeta):
self.storage = None
self.errors = []
self.data = None
+ return self
def __init__(self, name):
self._init()
self._load(name)
- # cache in prefix
- ProtocolTemplate[self.name] = self
-
@classmethod
def new(
- cls, sets, name, description=None, schema_version=None,
+ cls, sets, name, description=None, schema_version=1,
):
- self = cls.__new__(cls)
- self._init()
+ self = cls.__new__(cls)._init()
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
@@ -144,21 +142,17 @@ class ProtocolTemplate(metaclass=config.PrefixMeta):
k: dataformat_name(v) for k, v in set_["outputs"].items()
}
- data = dict(sets=sets)
+ data = dict(sets=sets, schema_version=schema_version)
if description is not None:
data["description"] = description
- if schema_version is not None:
- data["schema_version"] = schema_version
-
+ print("ProtocolTemplate data", data)
self.data = data
- self.storage = Storage(name)
+ self.storage = Storage(self.name)
+ self.storage.json.contents = str(self)
self._resolve()
- # cache in prefix
- ProtocolTemplate[self.name] = self
-
return self
def _resolve(self):
@@ -327,15 +321,11 @@ class ProtocolTemplate(metaclass=config.PrefixMeta):
if not self.valid:
raise RuntimeError("protocol template is not valid")
- if prefix == prefix:
- raise RuntimeError(
- "Cannot export protocol template to the same prefix (" "%s)" % prefix
- )
-
for k in self.dataformats.values():
k.export(prefix)
- self.write(Storage(prefix, self.name))
+ print("ProtocolTemplate data at export", self.data)
+ self.write(Storage(self.name, prefix=prefix))
def sets(self):
"""Returns all the sets available in this protocol template"""
diff --git a/beat/backend/python/test/test_interactive.py b/beat/backend/python/test/test_interactive.py
index 4f2a1da8e9ba407d65e058e35a5025eb0ca6a0ec..c45ab3f46e71bd48e0855afc10368b5cbacaf2b6 100644
--- a/beat/backend/python/test/test_interactive.py
+++ b/beat/backend/python/test/test_interactive.py
@@ -111,6 +111,18 @@ class DataFormatTest(unittest.TestCase):
with self.assertRaises(ValueError):
DataFormat.new({"value": int}, name="")
+ # test name change
+ df = DataFormat.new({"value": int}, name="int")
+ old_name = df.name
+ df.name = "int2"
+ new_name = df.name
+ self.assertTrue(old_name not in DataFormat)
+ self.assertTrue(new_name in DataFormat, new_name)
+
+ # test creation with init
+ df = DataFormat({"value": int})
+ self.assertTrue(len(df.errors) == 0, df.errors)
+
# class AlgorithmTest(unittest.TestCase):
# """docstring for AlgorithmTest"""
diff --git a/beat/backend/python/utils.py b/beat/backend/python/utils.py
index 4468a3f92499b645068181dc133415e1d0c46100..04662c89ab721ec0967629d4667613a17d4e19eb 100644
--- a/beat/backend/python/utils.py
+++ b/beat/backend/python/utils.py
@@ -48,7 +48,6 @@ import shutil
import numpy
import simplejson
-import six
from . import hash
@@ -132,6 +131,7 @@ class File(object):
self.path = path
self.binary = binary
+ self.contents = None
def exists(self):
@@ -139,12 +139,18 @@ class File(object):
def load(self):
- mode = "rb" if self.binary else "rt"
- with open(self.path, mode) as f:
- return f.read()
+ if self.contents is None:
+ mode = "rb" if self.binary else "rt"
+ with open(self.path, mode) as f:
+ self.contents = f.read()
+
+ return self.contents
def try_load(self):
+ if self.contents is not None:
+ return self.contents
+
if os.path.exists(self.path):
return self.load()
return None
@@ -172,11 +178,11 @@ class File(object):
mode = "wb" if self.binary else "wt"
if self.binary:
mode = "wb"
- if isinstance(contents, six.string_types):
+ if isinstance(contents, str):
contents = contents.encode("utf-8")
else:
mode = "wt"
- if not isinstance(contents, six.string_types):
+ if not isinstance(contents, str):
contents = contents.decode("utf-8")
with open(self.path, mode) as f:
@@ -247,8 +253,8 @@ class Storage(AbstractStorage):
def hash(self, description="description"):
"""Re-imp"""
-
- return hash.hashJSONFile(self.json.path, description)
+ contents = simplejson.loads(self.json.load())
+ return hash.hashJSON(contents=contents, description=description)
def load(self):
"""Re-imp"""
@@ -261,7 +267,7 @@ class Storage(AbstractStorage):
if description:
self.doc.save(description.encode("utf8"))
- if not isinstance(declaration, six.string_types):
+ if not isinstance(declaration, str):
declaration = simplejson.dumps(declaration, indent=4)
self.json.save(declaration)
@@ -309,10 +315,11 @@ class CodeStorage(AbstractStorage):
def hash(self):
"""Re-imp"""
- declaration_hash = hash.hashJSONFile(self.json.path, "description")
+ contents = simplejson.loads(self.json.load())
+ declaration_hash = hash.hashJSON(contents=contents, description="description")
if self.code.exists():
- code_hash = hash.hashFileContents(self.code.path)
+ code_hash = hash.hash(self.code.load())
return hash.hash(dict(declaration=declaration_hash, code=code_hash))
else:
return declaration_hash
@@ -336,7 +343,7 @@ class CodeStorage(AbstractStorage):
if description:
self.doc.save(description.encode("utf8"))
- if not isinstance(declaration, six.string_types):
+ if not isinstance(declaration, str):
declaration = simplejson.dumps(declaration, indent=4)
self.json.save(declaration)