Commit 8fbee90b authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Use a global prefix to keep things around, add an iris test

parent 73491015
Pipeline #42047 failed with stage
in 90 minutes and 5 seconds
This diff is collapsed.
......@@ -99,7 +99,7 @@ def set_config(**kwargs):
"""
supported_keys = set(DEFAULTS.keys())
set_keys = set(kwargs.keys())
if set_keys not in supported_keys:
if not set_keys.issubset(supported_keys):
raise ValueError(
f"Only {supported_keys} are valid configurations. "
f"Got these extra values: {set_keys - supported_keys}"
......@@ -137,14 +137,14 @@ def config_context(**new_config):
"""
old_config = get_config().copy()
# also backup prefix
old_prefix = Prefix().copy()
prefix = Prefix()
old_prefix = prefix.copy()
set_config(**new_config)
try:
yield
finally:
set_config(**old_config)
prefix = Prefix()
prefix.clear()
prefix.update(old_prefix)
......@@ -154,7 +154,6 @@ def config_context(**new_config):
class Singleton(type):
"""A Singleton metaclass
The singleton class calls the __init__ method each time the instance is requested.
From: https://stackoverflow.com/a/6798042/1286165
"""
......@@ -167,11 +166,55 @@ class Singleton(type):
class Prefix(dict, metaclass=Singleton):
def __init__(self, path=None, *args, **kwargs):
super().__init__(*args, **kwargs)
pass
class PrefixMeta(type):
# cache instances when __init__ is called
def __call__(cls, *args, **kwargs):
instance = super().__call__(*args, **kwargs)
folder = f"{cls.asset_folder}/{instance.name}"
prefix = Prefix()
prefix[folder] = instance
print(f"caching {cls.__name__}/{instance.name} in __init__ calls")
return instance
def __init__(cls, name, bases, clsdict):
# cache instances when new is called
if "new" in clsdict:
def caching_new(*args, **kwargs):
instance = clsdict["new"].__func__(cls, *args, **kwargs)
folder = f"{cls.asset_folder}/{instance.name}"
prefix = Prefix()
prefix[folder] = instance
print(f"caching {cls.__name__}/{instance.name} in new calls")
return instance
setattr(cls, "new", caching_new)
# change the key of instance when its name is changed
if "name" in clsdict:
name_property = clsdict["name"]
actual_fset = name_property.fset
def updating_fset(self, value):
print(f"updating name of {cls.__name__}/{self.name} in cached prefix")
if self.name in cls:
del cls[self.name]
actual_fset(self, value)
cls[self.name] = self
name_property = property(
name_property.fget,
updating_fset,
name_property.fdel,
name_property.__doc__,
)
setattr(cls, "name", name_property)
def __contains__(cls, key):
return f"{cls.asset_folder}/{key}" in Prefix()
......@@ -191,3 +234,8 @@ class PrefixMeta(type):
folder = f"{cls.asset_folder}/{key}"
prefix = Prefix()
prefix[folder] = value
def __delitem__(cls, key):
folder = f"{cls.asset_folder}/{key}"
prefix = Prefix()
del prefix[folder]
......@@ -351,7 +351,7 @@ class CachedDataSource(DataSource):
)
self.dataformat = algo.result_dataformat()
else:
self.dataformat = DataFormat(self.prefix, dataformat_name)
self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError(
......@@ -614,7 +614,7 @@ class DatabaseOutputDataSource(DataSource):
self.output_name = output_name
self.pack = pack
self.dataformat = DataFormat(self.prefix, dataformat_name)
self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)
......@@ -731,7 +731,7 @@ class RemoteDataSource(DataSource):
self.input_name = input_name
self.unpack = unpack
self.dataformat = DataFormat(prefix, dataformat_name)
self.dataformat = DataFormat[dataformat_name]
if not self.dataformat.valid:
raise RuntimeError("the dataformat `%s' is not valid" % dataformat_name)
......
......@@ -60,6 +60,9 @@ from .exceptions import OutputError
from .outputs import OutputList
from .protocoltemplate import ProtocolTemplate
DATABASE_TYPE = "database"
DATABASE_FOLDER = "databases"
# ----------------------------------------------------------
......@@ -73,20 +76,21 @@ class Storage(utils.CodeStorage):
"""
asset_type = "database"
asset_folder = "databases"
asset_type = DATABASE_TYPE
asset_folder = DATABASE_FOLDER
def __init__(self, name):
def __init__(self, name, prefix=None):
if name.count("/") != 1:
raise RuntimeError("invalid database name: `%s'" % name)
self.name, self.version = name.split("/")
self.fullname = name
if prefix is None:
prefix = config.get_config()["prefix"]
self.prefix = prefix
path = os.path.join(
config.get_config()["prefix"], self.asset_folder, name + ".json"
)
path = os.path.join(prefix, self.asset_folder, name + ".json")
path = path[:-5]
# views are coded in Python
super(Storage, self).__init__(path, "python")
......@@ -217,7 +221,7 @@ class Runner(object):
# ----------------------------------------------------------
class Database(object):
class Database(metaclass=config.PrefixMeta):
"""Databases define the start point of the dataflow in an experiment.
......@@ -234,12 +238,16 @@ class Database(object):
"""
asset_type = DATABASE_TYPE
asset_folder = DATABASE_FOLDER
def _init(self):
self._name = None
self.dataformats = {} # preloaded dataformats
self.storage = None
self.errors = []
self.data = None
return self
def __init__(self, name):
......@@ -256,8 +264,7 @@ class Database(object):
schema_version=2,
root_folder="/foo/bar",
):
self = cls.__new__(cls)
self._init()
self = cls.__new__(cls)._init()
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
......@@ -275,7 +282,7 @@ class Database(object):
for i, proto in enumerate(protocols):
protocols[i]["template"] = protocoltemplate_name(proto["template"])
data = dict(protocols=protocols)
data = dict(protocols=protocols, root_folder=root_folder)
if description is not None:
data["description"] = description
if schema_version is not None:
......@@ -283,12 +290,11 @@ class Database(object):
self.data = data
self.storage = Storage(name)
# save the code into storage
self.storage = Storage(self.name)
self.code_path = self.storage.code.path = code_path
with open(code_path, "rt") as f:
self.storage.code.save(f.read())
self.code_path = self.storage.code.path
self.code = self.storage.code.load()
self.code = self.storage.code.contents = f.read()
self.storage.json.contents = str(self)
self._load_v2()
......@@ -496,7 +502,8 @@ class Database(object):
protocol = self.protocol(protocol_name)
template_name = protocol["template"]
protocol_template = ProtocolTemplate[template_name]
view_definition = protocol_template.set(set_name)
# copy the set so we don't modify the original set
view_definition = dict(protocol_template.set(set_name))
view_definition["view"] = protocol["views"][set_name]["view"]
parameters = protocol["views"][set_name].get("parameters")
if parameters is not None:
......@@ -629,11 +636,6 @@ class Database(object):
if not self.valid:
raise RuntimeError("database is not valid")
if prefix == config.get_config()["prefix"]:
raise RuntimeError(
"Cannot export database to the same prefix (" "%s)" % prefix
)
for k in self.dataformats.values():
k.export(prefix)
......@@ -642,7 +644,7 @@ class Database(object):
protocol_template = ProtocolTemplate[protocol["template"]]
protocol_template.export(prefix)
self.write(Storage(self.name))
self.write(Storage(self.name, prefix=prefix))
# ----------------------------------------------------------
......
......@@ -54,8 +54,8 @@ from . import config
from . import utils
from .baseformat import baseformat
DATA_FORMAT_TYPE = "dataformat"
DATA_FORMAT_FOLDER = "dataformats"
DATAFORMAT_TYPE = "dataformat"
DATAFORMAT_FOLDER = "dataformats"
# ----------------------------------------------------------
......@@ -70,25 +70,27 @@ class Storage(utils.Storage):
"""
asset_type = DATA_FORMAT_TYPE
asset_folder = DATA_FORMAT_FOLDER
asset_type = DATAFORMAT_TYPE
asset_folder = DATAFORMAT_FOLDER
def __init__(self, name):
def __init__(self, name, prefix=None):
if name.count("/") != 2:
raise RuntimeError("invalid dataformat name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
prefix = config.get_config()["prefix"]
if prefix is None:
prefix = config.get_config()["prefix"]
self.prefix = prefix
path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
super(Storage, self).__init__(path)
super().__init__(path)
def hash(self):
"""The 64-character hash of the database declaration JSON"""
return super(Storage, self).hash("#description")
return super().hash("#description")
# ----------------------------------------------------------
......@@ -141,8 +143,8 @@ class DataFormat(metaclass=config.PrefixMeta):
"""
asset_type = DATA_FORMAT_TYPE
asset_folder = DATA_FORMAT_FOLDER
asset_type = DATAFORMAT_TYPE
asset_folder = DATAFORMAT_FOLDER
def _init(self):
self._name = None
......@@ -153,24 +155,23 @@ class DataFormat(metaclass=config.PrefixMeta):
self.resolved = None
self.referenced = {}
self.parent = None
return self
def __init__(self, data, parent=None):
self._init()
self.parent = parent
self._load(data)
# cache in prefix
DataFormat[self.name] = self
def _load(self, data):
"""Loads the dataformat"""
if isinstance(data, dict):
self._name = "analysis:result"
self.name = "analysis:result"
self.data = data
else:
self._name = data
self.storage = Storage(data)
self.name = data
self.storage = Storage(self.name)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append(
......@@ -264,8 +265,7 @@ class DataFormat(metaclass=config.PrefixMeta):
schema_version=None,
parent=None,
):
self = cls.__new__(cls)
self._init()
self = cls.__new__(cls)._init()
def str_or_dtype_or_type(v):
# if it is a dict
......@@ -301,21 +301,16 @@ class DataFormat(metaclass=config.PrefixMeta):
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
if name != "analysis:result" and "/" not in name:
name = f"{config.get_config()['user']}/{name}/1"
self._name = name
self.name = name
self.parent = parent
if name != "analysis:result":
self.storage = Storage(name)
if self.name != "analysis:result":
self.storage = Storage(self.name)
self.storage.json.contents = str(self)
self._resolve()
# cache in prefix
DataFormat[self.name] = self
return self
@property
......@@ -330,8 +325,11 @@ class DataFormat(metaclass=config.PrefixMeta):
@name.setter
def name(self, value):
if value != "analysis:result" and "/" not in value:
value = f"{config.get_config()['user']}/{value}/1"
self._name = value
self.storage = Storage(value)
if value != "analysis:result":
self.storage = Storage(value)
@property
def schema_version(self):
......@@ -578,12 +576,7 @@ class DataFormat(metaclass=config.PrefixMeta):
if not self.valid:
raise RuntimeError("dataformat is not valid:\n{}".format(self.errors))
if prefix == prefix:
raise RuntimeError(
"Cannot export dataformat to the same prefix (" "%s)" % prefix
)
for k in self.referenced.values():
k.export(prefix)
self.write(Storage(prefix, self.name))
self.write(Storage(self.name, prefix=prefix))
......@@ -181,6 +181,61 @@ class AlgorithmExecutor(object):
self.loop_channel = LoopChannel(self.loop_socket)
self.loop_channel.setup(self.algorithm, self.prefix)
@classmethod
def new(
cls,
data,
socket,
db_socket=None,
loop_socket=None,
databases=None,
cache_root="/cache",
):
self = cls.__new__(cls)
self.data = data
self.socket = socket
self.db_socket = db_socket
self.loop_socket = loop_socket
self.loop_channel = None
self._runner = None
self.prefix = None
# Load the algorithm
self.algorithm = self.data["algorithm"]
if db_socket:
db_access_mode = AccessMode.REMOTE
else:
db_access_mode = AccessMode.LOCAL
(self.input_list, self.data_loaders) = create_inputs_from_configuration(
self.data,
self.algorithm,
prefix=self.prefix,
cache_root=cache_root,
cache_access=AccessMode.LOCAL,
db_access=db_access_mode,
socket=self.db_socket,
databases=databases,
)
# Loads algorithm outputs
(self.output_list, _) = create_outputs_from_configuration(
self.data,
self.algorithm,
prefix=self.prefix,
cache_root=cache_root,
input_list=self.input_list,
data_loaders=self.data_loaders,
loop_socket=self.loop_socket,
)
if self.loop_socket:
self.loop_channel = LoopChannel(self.loop_socket)
self.loop_channel.setup(self.algorithm, self.prefix)
return self
@property
def runner(self):
"""Returns the algorithm runner
......
......@@ -48,7 +48,6 @@ import hashlib
import os
import simplejson
import six
# ----------------------------------------------------------
......@@ -57,11 +56,8 @@ def _sha256(s):
"""A python2/3 shortcut for :py:func:`haslib.sha256.hexdigest` to will
ensure that the given string is unicode before going further.
"""
if isinstance(s, six.string_types):
try:
s = six.u(s).encode("utf-8")
except Exception:
s = s.encode("utf-8")
if isinstance(s, str):
s = s.encode("utf-8")
return hashlib.sha256(s).hexdigest()
......
......@@ -46,9 +46,13 @@ import os
import simplejson as json
from . import config
from . import loader
from . import utils
LIBRARY_TYPE = "library"
LIBRARY_FOLDER = "libraries"
# ----------------------------------------------------------
......@@ -57,29 +61,26 @@ class Storage(utils.CodeStorage):
Parameters:
prefix (str): Establishes the prefix of
your installation.
name (str): The name of the library object in the format
``<user>/<name>/<version>``.
"""
asset_type = "library"
asset_folder = "libraries"
asset_type = LIBRARY_TYPE
asset_folder = LIBRARY_FOLDER
def __init__(self, prefix, name, language=None):
def __init__(self, name, language=None, prefix=None):
if name.count("/") != 2:
raise RuntimeError("invalid library name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
if prefix is None:
prefix = config.get_config()["prefix"]
self.prefix = prefix
path = utils.hashed_or_simple(
self.prefix, self.asset_folder, name, suffix=".json"
)
path = utils.hashed_or_simple(prefix, self.asset_folder, name, suffix=".json")
path = path[:-5]
super(Storage, self).__init__(path, language)
......@@ -87,7 +88,7 @@ class Storage(utils.CodeStorage):
# ----------------------------------------------------------
class Library(object):
class Library(metaclass=config.PrefixMeta):
"""Librarys represent independent algorithm components within the platform.
This class can only parse the meta-parameters of the library. The actual
......@@ -97,15 +98,8 @@ class Library(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
library_cache (:py:class:`dict`, Optional): A dictionary mapping library
names to loaded libraries. This parameter is optional and, if passed,
may greatly speed-up library loading times as libraries that are
already loaded may be re-used.
Attributes:
......@@ -136,28 +130,58 @@ class Library(object):
"""
def __init__(self, prefix, name, library_cache=None):
asset_type = LIBRARY_TYPE
asset_folder = LIBRARY_FOLDER
def _init(self):
self._name = None
self.storage = None
self.prefix = prefix
self.errors = []
self.libraries = {}
return self
def __init__(self, name):
self._init()
self._load(name)
@classmethod
def new(
cls, code_path, name, description=None, language="python", uses=None,
):
self = cls.__new__(cls)._init()
def lib_name(v):
if hasattr(v, "name"):
v = v.name
return v
library_cache = library_cache if library_cache is not None else {}
uses = {k: lib_name(v) for k, v in (uses or {}).items()} or None
data = dict(language=language, description=description, uses=uses)
data = {k: v for k, v in data.items() if v is not None}
try:
self._load(name, library_cache)
finally:
if self._name is not None: # registers it into the cache, even if failed
library_cache[self._name] = self
self.data = data
def _load(self, data, library_cache):
if not name:
raise ValueError(f"Invalid {name}. The name should be a non-empty string!")
self.name = name
self.storage = Storage(self.name)
self.code_path = self.storage.code.path = code_path
with open(code_path, "rt") as f:
self.code = self.storage.code.contents = f.read()
self.storage.json.contents = str(self)
self._resolve()
return self
def _load(self, data):
"""Loads the library"""
self._name = data
self.name = data
self.storage = Storage(self.prefix, data)
self.storage = Storage(self.name)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append("Library declaration file not found: %s" % json_path)
......@@ -178,11 +202,12 @@ class Library(object):
# if no errors so far, make sense out of the library data
self.data.setdefault("uses", {})
self._resolve()
def _resolve(self):
if self.uses is not None:
for name, value in self.uses.items():
self.libraries[value] = Library(self.prefix, value, library_cache)
self.libraries[self._name] = self
self.libraries[value] = Library[value]
def uses_dict(self):
"""Returns the usage dictionary for all dependent modules"""
......@@ -233,8 +258,11 @@ class Library(object):
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
if "/" not in value:
value = f"{config.get_config()['user']}/{value}/1"
self._name = value
self.storage = Storage(self.prefix, value, self.data["language"])
self.storage = Storage(value, self.data["language"])
@property
def schema_version(self):
......@@ -252,7 +280,6 @@ class Library(object):
if self.storage:
self.storage.language = value
self.data["language"] = value
self._check_language_consistence()
@property
def valid(self):
......@@ -369,7 +396,7 @@ class Library(object):
Raises:
RuntimeError: If prefix and self.prefix point to the same directory.
RuntimeError: If prefix and prefix point to the same directory.
"""
......@@ -379,12 +406,7 @@ class Library(object):
if not self.valid:
raise RuntimeError("library is not valid")
if prefix == self.prefix:
raise RuntimeError(
"Cannot export library to the same prefix (" "%s)" % (prefix)
)
for k in self.libraries.values():
k.export(prefix)
self.write(Storage(prefix, self.name))
self.write(Storage(self.name, prefix=prefix))
......@@ -48,8 +48,8 @@ from . import config
from . import utils
from .dataformat import DataFormat
PROTOCOL_TEMPLATE_TYPE = "protocoltemplate"
PROTOCOL_TEMPLATE_FOLDER = "protocoltemplates"
PROTOCOLTEMPLATE_TYPE = "protocoltemplate"
PROTOCOLTEMPLATE_FOLDER = "protocoltemplates"
# ----------------------------------------------------------
......@@ -63,20 +63,21 @@ class Storage(utils.Storage):
"""
asset_type = PROTOCOL_TEMPLATE_TYPE
asset_folder = PROTOCOL_TEMPLATE_FOLDER
asset_type = PROTOCOLTEMPLATE_TYPE
asset_folder = PROTOCOLTEMPLATE_FOLDER
def __init__(self, name):
def __init__(self, name, prefix=None):
if name.count("/") != 1:
raise RuntimeError("invalid protocol template name: `%s'" % name)
self.name, self.version = name.split("/")
self