Commit 49181315 authored by Flavio TARSETTI's avatar Flavio TARSETTI
Browse files

Merge branch 'improve_asset_information' into 'master'

Improve asset information

See merge request !46
parents b0200612 c6272e85
Pipeline #29913 passed with stages
in 6 minutes and 40 seconds
This diff is collapsed.
......@@ -75,6 +75,9 @@ class Storage(utils.CodeStorage):
"""
asset_type = "database"
asset_folder = "databases"
def __init__(self, prefix, name):
if name.count("/") != 1:
......@@ -84,7 +87,7 @@ class Storage(utils.CodeStorage):
self.fullname = name
self.prefix = prefix
path = os.path.join(self.prefix, "databases", name + ".json")
path = os.path.join(self.prefix, self.asset_folder, name + ".json")
path = path[:-5]
# views are coded in Python
super(Storage, self).__init__(path, "python")
......
......@@ -68,23 +68,27 @@ class Storage(utils.Storage):
"""
asset_type = "dataformat"
asset_folder = "dataformats"
def __init__(self, prefix, name):
if name.count('/') != 2:
if name.count("/") != 2:
raise RuntimeError("invalid dataformat name: `%s'" % name)
self.username, self.name, self.version = name.split('/')
self.username, self.name, self.version = name.split("/")
self.fullname = name
self.prefix = prefix
path = utils.hashed_or_simple(self.prefix, 'dataformats', name, suffix='.json')
path = utils.hashed_or_simple(
self.prefix, self.asset_folder, name, suffix=".json"
)
path = path[:-5]
super(Storage, self).__init__(path)
def hash(self):
"""The 64-character hash of the database declaration JSON"""
return super(Storage, self).hash('#description')
return super(Storage, self).hash("#description")
# ----------------------------------------------------------
......@@ -165,23 +169,24 @@ class DataFormat(object):
if self._name is not None: # registers it into the cache, even if failed
dataformat_cache[self._name] = self
def _load(self, data, dataformat_cache):
"""Loads the dataformat"""
if isinstance(data, dict):
self._name = 'analysis:result'
self._name = "analysis:result"
self.data = data
else:
self._name = data
self.storage = Storage(self.prefix, data)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append('Dataformat declaration file not found: %s' % json_path)
self.errors.append(
"Dataformat declaration file not found: %s" % json_path
)
return
with open(json_path, 'rb') as f:
self.data = simplejson.loads(f.read().decode('utf-8'))
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
dataformat_cache[self._name] = self # registers itself into the cache
......@@ -189,17 +194,20 @@ class DataFormat(object):
# remove reserved fields
def is_reserved(x):
'''Returns if the field name is a reserved name'''
return (x.startswith('__') and x.endswith('__')) or \
x in ('#description', '#schema_version')
"""Returns if the field name is a reserved name"""
return (x.startswith("__") and x.endswith("__")) or x in (
"#description",
"#schema_version",
)
for key in list(self.resolved):
if is_reserved(key): del self.resolved[key]
if is_reserved(key):
del self.resolved[key]
def maybe_load_format(name, obj, dataformat_cache):
"""Tries to load a given dataformat from its relative path"""
if isinstance(obj, six.string_types) and obj.find('/') != -1: # load it
if isinstance(obj, six.string_types) and obj.find("/") != -1: # load it
if obj in dataformat_cache: # reuse
......@@ -209,8 +217,9 @@ class DataFormat(object):
self.referenced[obj] = dataformat_cache[obj]
else: # load it
self.referenced[obj] = DataFormat(self.prefix, obj, (self, name),
dataformat_cache)
self.referenced[obj] = DataFormat(
self.prefix, obj, (self, name), dataformat_cache
)
return self.referenced[obj]
......@@ -226,22 +235,22 @@ class DataFormat(object):
# now checks that every referred dataformat is loaded, resolves it
for field, value in self.data.items():
if field in ('#description', '#schema_version'):
if field in ("#description", "#schema_version"):
continue # skip the description and schema version meta attributes
self.resolved[field] = maybe_load_format(field, value, dataformat_cache)
# at this point, there should be no more external references in
# ``self.resolved``. We treat the "#extends" property, which requires a
# special handling, given its nature.
if '#extends' in self.resolved:
if "#extends" in self.resolved:
ext = self.data['#extends']
ext = self.data["#extends"]
self.referenced[ext] = maybe_load_format(self._name, ext, dataformat_cache)
basetype = self.resolved['#extends']
basetype = self.resolved["#extends"]
tmp = self.resolved
self.resolved = basetype.resolved
self.resolved.update(tmp)
del self.resolved['#extends'] # avoids infinite recursion
del self.resolved["#extends"] # avoids infinite recursion
@property
def name(self):
......@@ -249,29 +258,25 @@ class DataFormat(object):
from the hierarchy it belongs.
"""
if self.parent and self._name is None:
return self.parent[0].name + '.' + self.parent[1] + '_type'
return self.parent[0].name + "." + self.parent[1] + "_type"
else:
return self._name or '__unnamed_dataformat__'
return self._name or "__unnamed_dataformat__"
@name.setter
def name(self, value):
self._name = value
self.storage = Storage(self.prefix, value)
@property
def schema_version(self):
"""Returns the schema version"""
return self.data.get('#schema_version', 1)
return self.data.get("#schema_version", 1)
@property
def extends(self):
"""If this dataformat extends another one, this is it, otherwise ``None``
"""
return self.data.get('#extends')
return self.data.get("#extends")
@property
def type(self):
......@@ -305,16 +310,14 @@ class DataFormat(object):
if self.resolved is None:
raise RuntimeError("Cannot prototype while not properly initialized")
classname = re.sub(r'[-/]', '_', self.name)
if not isinstance(classname, str): classname = str(classname)
classname = re.sub(r"[-/]", "_", self.name)
if not isinstance(classname, str):
classname = str(classname)
def init(self, **kwargs): baseformat.__init__(self, **kwargs)
def init(self, **kwargs):
baseformat.__init__(self, **kwargs)
attributes = dict(
__init__=init,
_name=self.name,
_format=self.resolved,
)
attributes = dict(__init__=init, _name=self.name, _format=self.resolved)
# create the converters for the class we're about to return
for k, v in self.resolved.items():
......@@ -324,7 +327,7 @@ class DataFormat(object):
if isinstance(v[-1], DataFormat):
attributes[k][-1] = v[-1].type
else:
if v[-1] in ('string', 'str'):
if v[-1] in ("string", "str"):
attributes[k][-1] = str
else:
attributes[k][-1] = numpy.dtype(v[-1])
......@@ -333,35 +336,27 @@ class DataFormat(object):
attributes[k] = v.type
else: # it is a simple type
if v in ('string', 'str'):
if v in ("string", "str"):
attributes[k] = str
else:
attributes[k] = numpy.dtype(v)
return type(
classname,
(baseformat,),
attributes,
)
return type(classname, (baseformat,), attributes)
@property
def valid(self):
"""A boolean that indicates if this dataformat is valid or not"""
return not bool(self.errors)
@property
def description(self):
"""The short description for this object"""
return self.data.get('#description', None)
return self.data.get("#description", None)
@description.setter
def description(self, value):
"""Sets the short description for this object"""
self.data['#description'] = value
self.data["#description"] = value
@property
def documentation(self):
......@@ -374,7 +369,6 @@ class DataFormat(object):
return self.storage.doc.load()
return None
@documentation.setter
def documentation(self, value):
"""Sets the full-length description for this object"""
......@@ -382,12 +376,11 @@ class DataFormat(object):
if not self._name:
raise RuntimeError("dataformat has no name")
if hasattr(value, 'read'):
if hasattr(value, "read"):
self.storage.doc.save(value.read())
else:
self.storage.doc.save(value)
def hash(self):
"""Returns the hexadecimal hash for its declaration"""
......@@ -396,7 +389,6 @@ class DataFormat(object):
return self.storage.hash()
def validate(self, data):
"""Validates a piece of data provided by the user
......@@ -421,13 +413,12 @@ class DataFormat(object):
obj = self.type()
if isinstance(data, dict):
obj.from_dict(data, casting='safe', add_defaults=False)
obj.from_dict(data, casting="safe", add_defaults=False)
elif isinstance(data, six.string_types):
obj.unpack(data)
else:
obj.unpack_from(data)
def isparent(self, other):
"""Tells if the other object extends self (directly or indirectly).
......@@ -451,7 +442,6 @@ class DataFormat(object):
return False
def json_dumps(self, indent=4):
"""Dumps the JSON declaration of this object in a string
......@@ -467,14 +457,11 @@ class DataFormat(object):
"""
return simplejson.dumps(self.data, indent=indent,
cls=utils.NumpyJSONEncoder)
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
def write(self, storage=None):
"""Writes contents to prefix location
......@@ -493,7 +480,6 @@ class DataFormat(object):
storage.save(str(self), self.description)
def export(self, prefix):
"""Recursively exports itself into another prefix
......@@ -523,8 +509,9 @@ class DataFormat(object):
raise RuntimeError("dataformat is not valid")
if prefix == self.prefix:
raise RuntimeError("Cannot export dataformat to the same prefix ("
"%s)" % prefix)
raise RuntimeError(
"Cannot export dataformat to the same prefix (" "%s)" % prefix
)
for k in self.referenced.values():
k.export(prefix)
......
......@@ -65,16 +65,21 @@ class Storage(utils.CodeStorage):
"""
asset_type = "library"
asset_folder = "libraries"
def __init__(self, prefix, name, language=None):
if name.count('/') != 2:
if name.count("/") != 2:
raise RuntimeError("invalid library name: `%s'" % name)
self.username, self.name, self.version = name.split('/')
self.username, self.name, self.version = name.split("/")
self.fullname = name
self.prefix = prefix
path = utils.hashed_or_simple(self.prefix, 'libraries', name, suffix='.json')
path = utils.hashed_or_simple(
self.prefix, self.asset_folder, name, suffix=".json"
)
path = path[:-5]
super(Storage, self).__init__(path, language)
......@@ -144,10 +149,9 @@ class Library(object):
try:
self._load(name, library_cache)
finally:
if self._name is not None: #registers it into the cache, even if failed
if self._name is not None: # registers it into the cache, even if failed
library_cache[self._name] = self
def _load(self, data, library_cache):
"""Loads the library"""
......@@ -156,16 +160,16 @@ class Library(object):
self.storage = Storage(self.prefix, data)
json_path = self.storage.json.path
if not self.storage.exists():
self.errors.append('Library declaration file not found: %s' % json_path)
self.errors.append("Library declaration file not found: %s" % json_path)
return
with open(json_path, 'rb') as f:
self.data = simplejson.loads(f.read().decode('utf-8'))
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
self.code_path = self.storage.code.path
# if no errors so far, make sense out of the library data
self.data.setdefault('uses', {})
self.data.setdefault("uses", {})
if self.uses is not None:
for name, value in self.uses.items():
......@@ -173,11 +177,10 @@ class Library(object):
self.libraries[self._name] = self
def uses_dict(self):
"""Returns the usage dictionary for all dependent modules"""
if self.data['language'] == 'unknown':
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
if not self._name:
......@@ -189,95 +192,85 @@ class Library(object):
for name, value in self.uses.items():
retval[name] = dict(
path=self.libraries[value].storage.code.path,
uses=self.libraries[value].uses_dict(),
)
path=self.libraries[value].storage.code.path,
uses=self.libraries[value].uses_dict(),
)
return retval
def load(self):
"""Loads the Python module for this library resolving all references
Returns the loaded Python module.
"""
if self.data['language'] == 'unknown':
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
if not self._name:
raise RuntimeError("library has no name")
return loader.load_module(self.name.replace(os.sep, '_'),
self.storage.code.path, self.uses_dict())
return loader.load_module(
self.name.replace(os.sep, "_"), self.storage.code.path, self.uses_dict()
)
@property
def name(self):
"""Returns the name of this object"""
return self._name or '__unnamed_library__'
return self._name or "__unnamed_library__"
@name.setter
def name(self, value):
if self.data['language'] == 'unknown':
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
self._name = value
self.storage = Storage(self.prefix, value, self.data['language'])
self.storage = Storage(self.prefix, value, self.data["language"])
@property
def schema_version(self):
"""Returns the schema version"""
return self.data.get('schema_version', 1)
return self.data.get("schema_version", 1)
@property
def language(self):
"""Returns the current language set for the library code"""
return self.data['language']
return self.data["language"]
@language.setter
def language(self, value):
"""Sets the current executable code programming language"""
if self.storage: self.storage.language = value
self.data['language'] = value
if self.storage:
self.storage.language = value
self.data["language"] = value
self._check_language_consistence()
@property
def valid(self):
"""A boolean that indicates if this library is valid or not"""
return not bool(self.errors)
@property
def uses(self):
return self.data.get('uses')
return self.data.get("uses")
@uses.setter
def uses(self, value):
self.data['uses'] = value
self.data["uses"] = value
return value
@property
def description(self):
"""The short description for this object"""
return self.data.get('description', None)
return self.data.get("description", None)
@description.setter
def description(self, value):
"""Sets the short description for this object"""
self.data['description'] = value
self.data["description"] = value
@property
def documentation(self):
......@@ -290,7 +283,6 @@ class Library(object):
return self.storage.doc.load()
return None
@documentation.setter
def documentation(self, value):
"""Sets the full-length description for this object"""
......@@ -298,12 +290,11 @@ class Library(object):
if not self._name:
raise RuntimeError("library has no name")
if hasattr(value, 'read'):
if hasattr(value, "read"):
self.storage.doc.save(value.read())
else:
self.storage.doc.save(value)
def hash(self):
"""Returns the hexadecimal hash for the current library"""
......@@ -312,7 +303,6 @@ class Library(object):
return self.storage.hash()
def json_dumps(self, indent=4):
"""Dumps the JSON declaration of this object in a string
......@@ -328,14 +318,11 @@ class Library(object):
"""
return simplejson.dumps(self.data, indent=indent,
cls=utils.NumpyJSONEncoder)
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
def write(self, storage=None):
"""Writes contents to prefix location.
......@@ -347,17 +334,16 @@ class Library(object):
"""
if self.data['language'] == 'unknown':
if self.data["language"] == "unknown":
raise RuntimeError("library has no programming language set")
if storage is None:
if not self._name:
raise RuntimeError("library has no name")
storage = self.storage #overwrite
storage = self.storage # overwrite
storage.save(str(self), self.code, self.description)
def export(self, prefix):
"""Recursively exports itself into another prefix
......@@ -387,8 +373,9 @@ class Library(object):
raise RuntimeError("library is not valid")
if prefix == self.prefix:
raise RuntimeError("Cannot export library to the same prefix ("
"%s)" % (prefix))
raise RuntimeError(
"Cannot export library to the same prefix (" "%s)" % (prefix)
)
for k in self.libraries.values():
k.export(prefix)
......
......@@ -64,6 +64,9 @@ class Storage(utils.Storage):
"""
asset_type = "protocoltemplate"
asset_folder = "protocoltemplates"
def __init__(self, prefix, name):
if name.count("/") != 1:
......@@ -74,7 +77,7 @@ class Storage(utils.Storage):
self.prefix = prefix
path = utils.hashed_or_simple(
self.prefix, "protocoltemplates", name,