diff --git a/beat/backend/python/algorithm.py b/beat/backend/python/algorithm.py index 7172b8040cb4128343be4662f4a28db94e461da9..072e2cc13a7a5ae869b02b64b587386901d6aa6e 100755 --- a/beat/backend/python/algorithm.py +++ b/beat/backend/python/algorithm.py @@ -49,7 +49,8 @@ class Storage(utils.CodeStorage): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The name of the algorithm object in the format ``<user>/<name>/<version>``. @@ -261,7 +262,8 @@ class Algorithm(object): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of your + installation. name (str): The fully qualified algorithm name (e.g. ``user/algo/1``) @@ -850,9 +852,12 @@ class Algorithm(object): if not self.valid: raise RuntimeError("algorithm is not valid") - if os.path.samefile(prefix, self.prefix): - raise RuntimeError("Cannot export algorithm to the same prefix (%s == " \ - "%s)" % (prefix, self.prefix)) + if isinstance(prefix, six.string_types): + prefix = utils.Prefix(prefix) + + if prefix.paths[0] in self.prefix.paths: + raise RuntimeError("Cannot export algorithm to the same prefix (%s in " \ + "%s)" % (prefix.paths[0], self.prefix.paths)) for k in self.libraries.values(): k.export(prefix) diff --git a/beat/backend/python/data.py b/beat/backend/python/data.py index ce0554f720ef0a8228639b10c51f30fdc6a266e5..951a1b887d309ea95ac9a5ca59a9ee1f56c3ce53 100755 --- a/beat/backend/python/data.py +++ b/beat/backend/python/data.py @@ -305,7 +305,8 @@ class CachedDataSource(DataSource): filename (str): Name of the file to read the data from - prefix (str, path): Path to the prefix where the dataformats are stored. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. start_index (int): The starting index (if not set or set to ``None``, the default, read data from the begin of file) @@ -489,7 +490,8 @@ class DatabaseOutputDataSource(DataSource): Parameters: - prefix (str, path): Path to the prefix where the dataformats are stored. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. start_index (int): The starting index (if not set or set to ``None``, the default, read data from the begin of file) @@ -609,7 +611,8 @@ class RemoteDataSource(DataSource): dataformat_name (str): Name of the data format. - prefix (str, path): Path to the prefix where the dataformats are stored. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. unpack (bool): Indicates if the data must be unpacked or not @@ -976,12 +979,12 @@ class CachedDataSink(DataSink): #---------------------------------------------------------- -def load_data_index(cache_prefix, hash_path): +def load_data_index(cache_root, hash_path): """Loads a cached-data index if it exists. Returns empty otherwise. Parameters: - cache_prefix (str): The path to the root of the cache directory + cache_root (str): The path to the root of the cache directory hash_path (str): The hashed path of the input you wish to load the indexes for, as it is returned by the utility function @@ -996,7 +999,7 @@ def load_data_index(cache_prefix, hash_path): """ name_no_extension = os.path.splitext(hash_path)[0] # remove .data - index_stem = os.path.join(cache_prefix, name_no_extension) + index_stem = os.path.join(cache_root, name_no_extension) index_glob = index_stem + '*.index' candidates = glob.glob(index_glob) diff --git a/beat/backend/python/database.py b/beat/backend/python/database.py index ad881fe41609256ebbbed8eced7956e2a1f08654..f9255f50d789181b963665caa93fc176c41cb5c1 100755 --- a/beat/backend/python/database.py +++ b/beat/backend/python/database.py @@ -52,7 +52,8 @@ class Storage(utils.CodeStorage): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The name of the database object in the format ``<name>/<version>``. @@ -67,7 +68,7 @@ class Storage(utils.CodeStorage): self.name, self.version = name.split('/') self.fullname = name - path = os.path.join(prefix, 'databases', name) + path = prefix.path(os.path.join('databases', name)) super(Storage, self).__init__(path, 'python') #views are coded in Python @@ -84,7 +85,8 @@ class Runner(object): module (module): The preloaded module containing the database views as returned by :py:func:`beat.core.loader.load_module`. - prefix (str, path): The prefix path for the current installation + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. root_folder (str, path): The path pointing to the root folder of this database @@ -177,11 +179,6 @@ class Runner(object): return loader.run(self.obj, 'get', self.exc, output, index) - def __getattr__(self, key): - '''Returns an attribute of the view - only called at last resort''' - return getattr(self.obj, key) - - def objects(self): return self.obj.objs @@ -195,7 +192,8 @@ class Database(object): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The fully qualified database name (e.g. ``db/1``) diff --git a/beat/backend/python/dataformat.py b/beat/backend/python/dataformat.py index b394494f7ecf337e64996b93512241286a3638b6..382ccc3185b3e34ee09e15b743696589014c7bd0 100755 --- a/beat/backend/python/dataformat.py +++ b/beat/backend/python/dataformat.py @@ -40,13 +40,16 @@ from . import utils from .baseformat import baseformat +#---------------------------------------------------------- + class Storage(utils.Storage): """Resolves paths for dataformats Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The name of the dataformat object in the format ``<user>/<name>/<version>``. @@ -70,13 +73,16 @@ class Storage(utils.Storage): return super(Storage, self).hash('#description') +#---------------------------------------------------------- + class DataFormat(object): """Data formats define the chunks of data that circulate between blocks. Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. data (str, dict): The fully qualified algorithm name (e.g. ``user/algo/1``) or a dictionary representing the data format (for analyzer results). @@ -480,7 +486,8 @@ class DataFormat(object): Parameters: - prefix (str): A path to a prefix that must different then my own. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. Returns: @@ -500,9 +507,12 @@ class DataFormat(object): if not self.valid: raise RuntimeError("dataformat is not valid") - if os.path.samefile(prefix, self.prefix): - raise RuntimeError("Cannot dataformat object to the same prefix (%s " \ - "== %s)" % (prefix, self.prefix)) + if isinstance(prefix, six.string_types): + prefix = utils.Prefix(prefix) + + if prefix.paths[0] in self.prefix.paths: + raise RuntimeError("Cannot export dataformat to the same prefix (%s in " \ + "%s)" % (prefix.paths[0], self.prefix.paths)) for k in self.referenced.values(): k.export(prefix) diff --git a/beat/backend/python/dbexecution.py b/beat/backend/python/dbexecution.py index 551027d5721efff30afc0a16f0c05904d94f96b2..93bed5a34521832ed0f13e0fb5df6337ecfcb303 100755 --- a/beat/backend/python/dbexecution.py +++ b/beat/backend/python/dbexecution.py @@ -46,7 +46,8 @@ class DBExecutor(object): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. data (dict, str): The piece of data representing the block to be executed. It must validate against the schema defined for execution blocks. If a diff --git a/beat/backend/python/executor.py b/beat/backend/python/executor.py index 6a42035687f9c569f7709c36ad6ffd5221f65f79..640841c163412185e1d5d318b1b7e883ff7e9429 100755 --- a/beat/backend/python/executor.py +++ b/beat/backend/python/executor.py @@ -41,6 +41,7 @@ from .algorithm import Algorithm from .helpers import create_inputs_from_configuration from .helpers import create_outputs_from_configuration from .helpers import AccessMode +from .utils import Prefix from . import stats @@ -85,7 +86,7 @@ class Executor(object): with open(self.configuration, 'rb') as f: self.data = simplejson.load(f) - self.prefix = os.path.join(directory, 'prefix') + self.prefix = Prefix(os.path.join(directory, 'prefix')) self.runner = None # Temporary caches, if the user has not set them, for performance diff --git a/beat/backend/python/library.py b/beat/backend/python/library.py index 51c41aa91af0b4145e2b1c1689d19fdc01f9a2a9..f2394464563ebe885f9e910bd9ae75af9cf9b747 100755 --- a/beat/backend/python/library.py +++ b/beat/backend/python/library.py @@ -36,13 +36,16 @@ from . import loader from . import utils +#---------------------------------------------------------- + class Storage(utils.CodeStorage): """Resolves paths for libraries Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The name of the library object in the format ``<user>/<name>/<version>``. @@ -62,6 +65,8 @@ class Storage(utils.CodeStorage): super(Storage, self).__init__(path, language) +#---------------------------------------------------------- + class Library(object): """Librarys represent independent algorithm components within the platform. @@ -73,7 +78,8 @@ class Library(object): Parameters: - prefix (str): Establishes the prefix of your installation. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. name (str): The fully qualified algorithm name (e.g. ``user/algo/1``) @@ -346,7 +352,8 @@ class Library(object): Parameters: - prefix (str): A path to a prefix that must different then my own. + prefix (beat.backend.python.utils.Prefix): Establishes the prefix of + your installation. Returns: @@ -366,9 +373,12 @@ class Library(object): if not self.valid: raise RuntimeError("library is not valid") - if os.path.samefile(prefix, self.prefix): - raise RuntimeError("Cannot library object to the same prefix (%s == " \ - "%s)" % (prefix, self.prefix)) + if isinstance(prefix, six.string_types): + prefix = utils.Prefix(prefix) + + if prefix.paths[0] in self.prefix.paths: + raise RuntimeError("Cannot export library to the same prefix (%s in " \ + "%s)" % (prefix.paths[0], self.prefix.paths)) for k in self.libraries.values(): k.export(prefix) diff --git a/beat/backend/python/loader.py b/beat/backend/python/loader.py index 9b8efbcb707081b3e4129dfea939c35becdd9e39..205e2626f5d1647f29e1da1f6b3cd38cb1bef0ff 100755 --- a/beat/backend/python/loader.py +++ b/beat/backend/python/loader.py @@ -34,6 +34,9 @@ import six import imp +#---------------------------------------------------------- + + def load_module(name, path, uses): '''Loads the Python file as module, returns a proper Python module diff --git a/beat/backend/python/test/__init__.py b/beat/backend/python/test/__init__.py index 5af195d2ab4fda6fa19a56cebdf4a48a7d3646eb..e292e437ae9ac19e5648dd30bcffe0ef95e26beb 100644 --- a/beat/backend/python/test/__init__.py +++ b/beat/backend/python/test/__init__.py @@ -30,8 +30,10 @@ import pkg_resources import tempfile import sys +from ..utils import Prefix -prefix = pkg_resources.resource_filename(__name__, 'prefix') + +prefix = Prefix(pkg_resources.resource_filename(__name__, 'prefix')) if sys.platform == 'darwin': tmp_prefix = tempfile.mkdtemp(prefix=__name__, suffix='.tmpdir', dir='/tmp') diff --git a/beat/backend/python/utils.py b/beat/backend/python/utils.py index 04f096f29d52af7e07a9cc915d43857591210aba..901742bb18a247428db2472f31854ad4fd0af261 100755 --- a/beat/backend/python/utils.py +++ b/beat/backend/python/utils.py @@ -37,18 +37,20 @@ import six from . import hash +#---------------------------------------------------------- + def hashed_or_simple(prefix, what, path, suffix='.json'): """Returns a hashed path or simple path depending on where the resource is""" username, right_bit = path.split('/', 1) hashed_prefix = hash.toUserPath(username) - candidate = os.path.join(prefix, what, hashed_prefix, right_bit) + candidate = prefix.path(os.path.join(what, hashed_prefix, right_bit)) if os.path.exists(candidate + suffix): return candidate - return os.path.join(prefix, what, path) + return prefix.path(os.path.join(what, path)) #---------------------------------------------------------- @@ -57,7 +59,8 @@ def hashed_or_simple(prefix, what, path, suffix='.json'): def safe_rmfile(f): """Safely removes a file from the disk""" - if os.path.exists(f): os.unlink(f) + if os.path.exists(f): + os.unlink(f) #---------------------------------------------------------- @@ -67,8 +70,12 @@ def safe_rmdir(f): """Safely removes the directory containg a given file from the disk""" d = os.path.dirname(f) - if not os.path.exists(d): return - if not os.listdir(d): os.rmdir(d) + + if not os.path.exists(d): + return + + if not os.listdir(d): + os.rmdir(d) #---------------------------------------------------------- @@ -97,12 +104,37 @@ def extension_for_language(language): """ return dict( - unknown = '', - cxx = '.so', - matlab = '.m', - python = '.py', - r = '.r', - )[language] + unknown = '', + cxx = '.so', + matlab = '.m', + python = '.py', + r = '.r', + )[language] + + +#---------------------------------------------------------- + + +class Prefix(object): + + def __init__(self, paths=None): + if isinstance(paths, list): + self.paths = paths + elif paths is not None: + self.paths = [paths] + else: + self.paths = [] + + def add(self, path): + self.paths.append(path) + + def path(self, filename): + for p in self.paths: + fullpath = os.path.join(p, filename) + if os.path.exists(fullpath): + return fullpath + + return os.path.join(self.paths[0], filename) #---------------------------------------------------------- @@ -126,7 +158,8 @@ class File(object): def load(self): mode = 'rb' if self.binary else 'rt' - with open(self.path, mode) as f: return f.read() + with open(self.path, mode) as f: + return f.read() def try_load(self): @@ -138,21 +171,28 @@ class File(object): def backup(self): - if not os.path.exists(self.path): return #no point in backing-up + if not os.path.exists(self.path): + return #no point in backing-up + backup = self.path + '~' - if os.path.exists(backup): os.remove(backup) + if os.path.exists(backup): + os.remove(backup) + shutil.copy(self.path, backup) def save(self, contents): d = os.path.dirname(self.path) - if not os.path.exists(d): os.makedirs(d) + if not os.path.exists(d): + os.makedirs(d) - if os.path.exists(self.path): self.backup() + if os.path.exists(self.path): + self.backup() mode = 'wb' if self.binary else 'wt' - with open(self.path, mode) as f: f.write(contents) + with open(self.path, mode) as f: + f.write(contents) def remove(self): @@ -166,16 +206,7 @@ class File(object): class Storage(object): - """Resolves paths for objects that provide only a description - - Parameters: - - prefix (str): Establishes the prefix of your installation. - - name (str): The name of the database object in the format - ``<name>/<version>``. - - """ + """Resolves paths for objects that provide only a description""" def __init__(self, path): @@ -217,11 +248,6 @@ class CodeStorage(object): Parameters: - prefix (str): Establishes the prefix of your installation. - - name (str): The name of the database object in the format - ``<name>/<version>``. - language (str): One of the valdid programming languages """ @@ -262,11 +288,11 @@ class CodeStorage(object): return hash.hash(dict( json=hash.hashJSONFile(self.json.path, 'description'), code=hash.hashFileContents(self.code.path), - )) + )) else: return hash.hash(dict( json=hash.hashJSONFile(self.json.path, 'description'), - )) + )) def exists(self): """If the database declaration file exists"""