diff --git a/beat/backend/python/algorithm.py b/beat/backend/python/algorithm.py index 2b1d14b58126b404424f7c08b81a1c6b73d35448..5341a08551407c4c1ebcc9374ab67687b110559c 100644 --- a/beat/backend/python/algorithm.py +++ b/beat/backend/python/algorithm.py @@ -38,6 +38,34 @@ import simplejson from . import dataformat from . import library from . import loader +from . import utils + + + +class Storage(utils.CodeStorage): + """Resolves paths for algorithms + + Parameters: + + prefix (str): Establishes the prefix of your installation. + + name (str): The name of the algorithm object in the format + ``<user>/<name>/<version>``. + + """ + + def __init__(self, prefix, name, language=None): + + if name.count('/') != 2: + raise RuntimeError("invalid algorithm name: `%s'" % name) + + self.username, self.name, self.version = name.split('/') + self.prefix = prefix + self.fullname = name + + path = utils.hashed_or_simple(self.prefix, 'algorithms', name) + super(Storage, self).__init__(path, language) + class Runner(object): @@ -160,6 +188,7 @@ class Runner(object): return getattr(self.obj, key) + class Algorithm(object): """Algorithms represent runnable components within the platform. @@ -222,6 +251,9 @@ class Algorithm(object): groups (dict): A list containing dictionaries with inputs and outputs belonging to the same synchronization group. + errors (list): A list containing errors found while loading this + algorithm. + data (dict): The original data for this algorithm, as loaded by our JSON decoder. @@ -232,20 +264,34 @@ class Algorithm(object): def __init__(self, prefix, name, dataformat_cache=None, library_cache=None): + self._name = None + self.storage = None self.prefix = prefix self.dataformats = {} self.libraries = {} - self.groups = [] dataformat_cache = dataformat_cache if dataformat_cache is not None else {} library_cache = library_cache if library_cache is not None else {} - self.name = name - json_path = os.path.join(prefix, 'algorithms', name + '.json') - with open(json_path, 'rb') as f: self.data = simplejson.load(f) + self._load(name, dataformat_cache, library_cache) - self.code_path = os.path.join(prefix, 'algorithms', name + '.py') + + def _load(self, data, dataformat_cache, library_cache): + """Loads the algorithm""" + + self._name = data + + self.storage = Storage(self.prefix, data) + json_path = self.storage.json.path + if not self.storage.exists(): + self.errors.append('Algorithm declaration file not found: %s' % json_path) + return + + with open(json_path, 'rb') as f: + self.data = simplejson.load(f) + + self.code_path = self.storage.code.path self.groups = self.data['groups'] @@ -375,6 +421,22 @@ class Algorithm(object): library.Library(self.prefix, value, library_cache)) + @property + def name(self): + """Returns the name of this object + """ + return self._name or '__unnamed_algorithm__' + + + @name.setter + def name(self, value): + + if self.data['language'] == 'unknown': + raise RuntimeError("algorithm has no programming language set") + + self._name = value + self.storage = Storage(self.prefix, value, self.data['language']) + @property def schema_version(self): @@ -382,6 +444,20 @@ class Algorithm(object): return self.data.get('schema_version', 1) + @property + def language(self): + """Returns the current language set for the executable code""" + return self.data['language'] + + + @language.setter + def language(self, value): + """Sets the current executable code programming language""" + if self.storage: + self.storage.language = value + self.data['language'] = value + + def clean_parameter(self, parameter, value): """Checks if a given value against a declared parameter @@ -410,8 +486,8 @@ class Algorithm(object): ValueError: If the parameter cannot be safe cast into the algorithm's type. Alternatively, a ``ValueError`` may also be raised if a range or - choice was specified and the value does not obbey those settings - estipulated for the parameter + choice was specified and the value does not obey those settings + stipulated for the parameter """ @@ -437,35 +513,72 @@ class Algorithm(object): return retval + @property + def valid(self): + """A boolean that indicates if this algorithm is valid or not""" + + return not bool(self.errors) + + @property def uses(self): return self.data.get('uses') + @uses.setter + def uses(self, value): + self.data['uses'] = value + return value + + @property def results(self): return self.data.get('results') + @results.setter + def results(self, value): + self.data['results'] = value + return value + + @property def parameters(self): return self.data.get('parameters') + @parameters.setter + def parameters(self, value): + self.data['parameters'] = value + return value + + @property def splittable(self): return self.data.get('splittable', False) + @splittable.setter + def splittable(self, value): + self.data['splittable'] = value + return value + + def uses_dict(self): """Returns the usage dictionary for all dependent modules""" + if self.data['language'] == 'unknown': + raise RuntimeError("algorithm has no programming language set") + + if not self._name: + raise RuntimeError("algorithm has no name") + retval = {} if self.uses is not None: for name, value in self.uses.items(): retval[name] = dict( - path=self.libraries[value].code_path, + path=self.libraries[value].storage.code.path, uses=self.libraries[value].uses_dict(), ) @@ -489,11 +602,24 @@ class Algorithm(object): before using the ``process`` method. """ + if not self._name: + exc = exc or RuntimeError + raise exc("algorithm has no name") + + if self.data['language'] == 'unknown': + exc = exc or RuntimeError + raise exc("algorithm has no programming language set") + + if not self.valid: + message = "cannot load code for invalid algorithm (%s)" % (self.name,) + exc = exc or RuntimeError + raise exc(message) + # loads the module only once through the lifetime of the algorithm object try: self.__module = getattr(self, 'module', loader.load_module(self.name.replace(os.sep, '_'), - self.code_path, self.uses_dict())) + self.storage.code.path, self.uses_dict())) except Exception as e: if exc is not None: type, value, traceback = sys.exc_info() @@ -504,6 +630,52 @@ class Algorithm(object): return Runner(self.__module, klass, self, exc) + @property + def description(self): + """The short description for this object""" + return self.data.get('description', None) + + + @description.setter + def description(self, value): + """Sets the short description for this object""" + self.data['description'] = value + + + @property + def documentation(self): + """The full-length description for this object""" + + if not self._name: + raise RuntimeError("algorithm has no name") + + if self.storage.doc.exists(): + return self.storage.doc.load() + return None + + + @documentation.setter + def documentation(self, value): + """Sets the full-length description for this object""" + + if not self._name: + raise RuntimeError("algorithm has no name") + + if hasattr(value, 'read'): + self.storage.doc.save(value.read()) + else: + self.storage.doc.save(value) + + + def hash(self): + """Returns the hexadecimal hash for the current algorithm""" + + if not self._name: + raise RuntimeError("algorithm has no name") + + return self.storage.hash() + + def result_dataformat(self): """Generates, on-the-fly, the dataformat for the result readout""" diff --git a/beat/backend/python/library.py b/beat/backend/python/library.py index 389d88232da05f95e9bca9bf7d378d680e246ab4..df126a5ae1785c432b2aa1e5effc910784626737 100644 --- a/beat/backend/python/library.py +++ b/beat/backend/python/library.py @@ -33,6 +33,34 @@ import os import simplejson from . import loader +from . import utils + + + +class Storage(utils.CodeStorage): + """Resolves paths for libraries + + Parameters: + + prefix (str): Establishes the prefix of your installation. + + name (str): The name of the library object in the format + ``<user>/<name>/<version>``. + + """ + + def __init__(self, prefix, name, language=None): + + if name.count('/') != 2: + raise RuntimeError("invalid library name: `%s'" % name) + + self.username, self.name, self.version = name.split('/') + self.prefix = prefix + self.fullname = name + + path = utils.hashed_or_simple(self.prefix, 'libraries', name) + super(Storage, self).__init__(path, language) + class Library(object): @@ -59,12 +87,23 @@ class Library(object): name (str): The library name + description (str): The short description string, loaded from the JSON + file if one was set. + + documentation (str): The full-length docstring for this object. + + storage (object): A simple object that provides information about file + paths for this library + libraries (dict): A mapping object defining other libraries this library needs to load so it can work properly. uses (dict): A mapping object defining the required library import name (keys) and the full-names (values). + errors (list): A list containing errors found while loading this + library. + data (dict): The original data for this library, as loaded by our JSON decoder. @@ -75,17 +114,36 @@ class Library(object): def __init__(self, prefix, name, library_cache=None): + self._name = None + self.storage = None self.prefix = prefix self.libraries = {} library_cache = library_cache if library_cache is not None else {} - self.name = name - json_path = os.path.join(prefix, 'libraries', name + '.json') - with open(json_path, 'rb') as f: self.data = simplejson.load(f) + try: + self._load(name, library_cache) + finally: + if self._name is not None: #registers it into the cache, even if failed + library_cache[self._name] = self + + + def _load(self, data, library_cache): + """Loads the library""" + + self._name = name + + self.storage = Storage(self.prefix, data) + json_path = self.storage.json.path + if not self.storage.exists(): + self.errors.append('Library declaration file not found: %s' % json_path) + return + + with open(json_path, 'rb') as f: + self.data = simplejson.load(f) - self.code_path = os.path.join(prefix, 'libraries', name + '.py') + self.code_path = self.storage.code.path # if no errors so far, make sense out of the library data self.data.setdefault('uses', {}) @@ -94,19 +152,25 @@ class Library(object): for name, value in self.uses.items(): self.libraries[value] = Library(self.prefix, value, library_cache) - self.libraries[self.name] = self + self.libraries[self._name] = self def uses_dict(self): """Returns the usage dictionary for all dependent modules""" + if self.data['language'] == 'unknown': + raise RuntimeError("library has no programming language set") + + if not self._name: + raise RuntimeError("library has no name") + retval = {} if self.uses is not None: for name, value in self.uses.items(): retval[name] = dict( - path=self.libraries[value].code_path, + path=self.libraries[value].storage.code.path, uses=self.libraries[value].uses_dict(), ) @@ -119,8 +183,31 @@ class Library(object): Returns the loaded Python module. """ + if self.data['language'] == 'unknown': + raise RuntimeError("library has no programming language set") + + if not self._name: + raise RuntimeError("library has no name") + return loader.load_module(self.name.replace(os.sep, '_'), - self.code_path, self.uses_dict()) + self.storage.code.path, self.uses_dict()) + + + @property + def name(self): + """Returns the name of this object + """ + return self._name or '__unnamed_library__' + + + @name.setter + def name(self, value): + + if self.data['language'] == 'unknown': + raise RuntimeError("library has no programming language set") + + self._name = value + self.storage = Storage(self.prefix, value, self.data['language']) @property @@ -129,6 +216,79 @@ class Library(object): return self.data.get('schema_version', 1) + @property + def language(self): + """Returns the current language set for the library code""" + return self.data['language'] + + + @language.setter + def language(self, value): + """Sets the current executable code programming language""" + if self.storage: self.storage.language = value + self.data['language'] = value + self._check_language_consistence() + + + @property + def valid(self): + """A boolean that indicates if this library is valid or not""" + + return not bool(self.errors) + + @property def uses(self): return self.data.get('uses') + + + @uses.setter + def uses(self, value): + self.data['uses'] = value + return value + + + @property + def description(self): + """The short description for this object""" + return self.data.get('description', None) + + + @description.setter + def description(self, value): + """Sets the short description for this object""" + self.data['description'] = value + + + @property + def documentation(self): + """The full-length description for this object""" + + if not self._name: + raise RuntimeError("library has no name") + + if self.storage.doc.exists(): + return self.storage.doc.load() + return None + + + @documentation.setter + def documentation(self, value): + """Sets the full-length description for this object""" + + if not self._name: + raise RuntimeError("library has no name") + + if hasattr(value, 'read'): + self.storage.doc.save(value.read()) + else: + self.storage.doc.save(value) + + + def hash(self): + """Returns the hexadecimal hash for the current library""" + + if not self._name: + raise RuntimeError("library has no name") + + return self.storage.hash()