diff --git a/beat/backend/python/algorithm.py b/beat/backend/python/algorithm.py
index 7172b8040cb4128343be4662f4a28db94e461da9..072e2cc13a7a5ae869b02b64b587386901d6aa6e 100755
--- a/beat/backend/python/algorithm.py
+++ b/beat/backend/python/algorithm.py
@@ -49,7 +49,8 @@ class Storage(utils.CodeStorage):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
@@ -261,7 +262,8 @@ class Algorithm(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of your
+ installation.
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
@@ -850,9 +852,12 @@ class Algorithm(object):
if not self.valid:
raise RuntimeError("algorithm is not valid")
- if os.path.samefile(prefix, self.prefix):
- raise RuntimeError("Cannot export algorithm to the same prefix (%s == " \
- "%s)" % (prefix, self.prefix))
+ if isinstance(prefix, six.string_types):
+ prefix = utils.Prefix(prefix)
+
+ if prefix.paths[0] in self.prefix.paths:
+ raise RuntimeError("Cannot export algorithm to the same prefix (%s in " \
+ "%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.libraries.values():
k.export(prefix)
diff --git a/beat/backend/python/data.py b/beat/backend/python/data.py
index ce0554f720ef0a8228639b10c51f30fdc6a266e5..951a1b887d309ea95ac9a5ca59a9ee1f56c3ce53 100755
--- a/beat/backend/python/data.py
+++ b/beat/backend/python/data.py
@@ -305,7 +305,8 @@ class CachedDataSource(DataSource):
filename (str): Name of the file to read the data from
- prefix (str, path): Path to the prefix where the dataformats are stored.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
start_index (int): The starting index (if not set or set to
``None``, the default, read data from the begin of file)
@@ -489,7 +490,8 @@ class DatabaseOutputDataSource(DataSource):
Parameters:
- prefix (str, path): Path to the prefix where the dataformats are stored.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
start_index (int): The starting index (if not set or set to
``None``, the default, read data from the begin of file)
@@ -609,7 +611,8 @@ class RemoteDataSource(DataSource):
dataformat_name (str): Name of the data format.
- prefix (str, path): Path to the prefix where the dataformats are stored.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
unpack (bool): Indicates if the data must be unpacked or not
@@ -976,12 +979,12 @@ class CachedDataSink(DataSink):
#----------------------------------------------------------
-def load_data_index(cache_prefix, hash_path):
+def load_data_index(cache_root, hash_path):
"""Loads a cached-data index if it exists. Returns empty otherwise.
Parameters:
- cache_prefix (str): The path to the root of the cache directory
+ cache_root (str): The path to the root of the cache directory
hash_path (str): The hashed path of the input you wish to load the indexes
for, as it is returned by the utility function
@@ -996,7 +999,7 @@ def load_data_index(cache_prefix, hash_path):
"""
name_no_extension = os.path.splitext(hash_path)[0] # remove .data
- index_stem = os.path.join(cache_prefix, name_no_extension)
+ index_stem = os.path.join(cache_root, name_no_extension)
index_glob = index_stem + '*.index'
candidates = glob.glob(index_glob)
diff --git a/beat/backend/python/database.py b/beat/backend/python/database.py
index ad881fe41609256ebbbed8eced7956e2a1f08654..f9255f50d789181b963665caa93fc176c41cb5c1 100755
--- a/beat/backend/python/database.py
+++ b/beat/backend/python/database.py
@@ -52,7 +52,8 @@ class Storage(utils.CodeStorage):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
@@ -67,7 +68,7 @@ class Storage(utils.CodeStorage):
self.name, self.version = name.split('/')
self.fullname = name
- path = os.path.join(prefix, 'databases', name)
+ path = prefix.path(os.path.join('databases', name))
super(Storage, self).__init__(path, 'python') #views are coded in Python
@@ -84,7 +85,8 @@ class Runner(object):
module (module): The preloaded module containing the database views as
returned by :py:func:`beat.core.loader.load_module`.
- prefix (str, path): The prefix path for the current installation
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
root_folder (str, path): The path pointing to the root folder of this
database
@@ -177,11 +179,6 @@ class Runner(object):
return loader.run(self.obj, 'get', self.exc, output, index)
- def __getattr__(self, key):
- '''Returns an attribute of the view - only called at last resort'''
- return getattr(self.obj, key)
-
-
def objects(self):
return self.obj.objs
@@ -195,7 +192,8 @@ class Database(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The fully qualified database name (e.g. ``db/1``)
diff --git a/beat/backend/python/dataformat.py b/beat/backend/python/dataformat.py
index b394494f7ecf337e64996b93512241286a3638b6..382ccc3185b3e34ee09e15b743696589014c7bd0 100755
--- a/beat/backend/python/dataformat.py
+++ b/beat/backend/python/dataformat.py
@@ -40,13 +40,16 @@ from . import utils
from .baseformat import baseformat
+#----------------------------------------------------------
+
class Storage(utils.Storage):
"""Resolves paths for dataformats
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The name of the dataformat object in the format
``<user>/<name>/<version>``.
@@ -70,13 +73,16 @@ class Storage(utils.Storage):
return super(Storage, self).hash('#description')
+#----------------------------------------------------------
+
class DataFormat(object):
"""Data formats define the chunks of data that circulate between blocks.
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
data (str, dict): The fully qualified algorithm name (e.g. ``user/algo/1``)
or a dictionary representing the data format (for analyzer results).
@@ -480,7 +486,8 @@ class DataFormat(object):
Parameters:
- prefix (str): A path to a prefix that must different then my own.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
Returns:
@@ -500,9 +507,12 @@ class DataFormat(object):
if not self.valid:
raise RuntimeError("dataformat is not valid")
- if os.path.samefile(prefix, self.prefix):
- raise RuntimeError("Cannot dataformat object to the same prefix (%s " \
- "== %s)" % (prefix, self.prefix))
+ if isinstance(prefix, six.string_types):
+ prefix = utils.Prefix(prefix)
+
+ if prefix.paths[0] in self.prefix.paths:
+ raise RuntimeError("Cannot export dataformat to the same prefix (%s in " \
+ "%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.referenced.values():
k.export(prefix)
diff --git a/beat/backend/python/dbexecution.py b/beat/backend/python/dbexecution.py
index 551027d5721efff30afc0a16f0c05904d94f96b2..93bed5a34521832ed0f13e0fb5df6337ecfcb303 100755
--- a/beat/backend/python/dbexecution.py
+++ b/beat/backend/python/dbexecution.py
@@ -46,7 +46,8 @@ class DBExecutor(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
data (dict, str): The piece of data representing the block to be executed.
It must validate against the schema defined for execution blocks. If a
diff --git a/beat/backend/python/executor.py b/beat/backend/python/executor.py
index 6a42035687f9c569f7709c36ad6ffd5221f65f79..640841c163412185e1d5d318b1b7e883ff7e9429 100755
--- a/beat/backend/python/executor.py
+++ b/beat/backend/python/executor.py
@@ -41,6 +41,7 @@ from .algorithm import Algorithm
from .helpers import create_inputs_from_configuration
from .helpers import create_outputs_from_configuration
from .helpers import AccessMode
+from .utils import Prefix
from . import stats
@@ -85,7 +86,7 @@ class Executor(object):
with open(self.configuration, 'rb') as f:
self.data = simplejson.load(f)
- self.prefix = os.path.join(directory, 'prefix')
+ self.prefix = Prefix(os.path.join(directory, 'prefix'))
self.runner = None
# Temporary caches, if the user has not set them, for performance
diff --git a/beat/backend/python/library.py b/beat/backend/python/library.py
index 51c41aa91af0b4145e2b1c1689d19fdc01f9a2a9..f2394464563ebe885f9e910bd9ae75af9cf9b747 100755
--- a/beat/backend/python/library.py
+++ b/beat/backend/python/library.py
@@ -36,13 +36,16 @@ from . import loader
from . import utils
+#----------------------------------------------------------
+
class Storage(utils.CodeStorage):
"""Resolves paths for libraries
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The name of the library object in the format
``<user>/<name>/<version>``.
@@ -62,6 +65,8 @@ class Storage(utils.CodeStorage):
super(Storage, self).__init__(path, language)
+#----------------------------------------------------------
+
class Library(object):
"""Librarys represent independent algorithm components within the platform.
@@ -73,7 +78,8 @@ class Library(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
@@ -346,7 +352,8 @@ class Library(object):
Parameters:
- prefix (str): A path to a prefix that must different then my own.
+ prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
+ your installation.
Returns:
@@ -366,9 +373,12 @@ class Library(object):
if not self.valid:
raise RuntimeError("library is not valid")
- if os.path.samefile(prefix, self.prefix):
- raise RuntimeError("Cannot library object to the same prefix (%s == " \
- "%s)" % (prefix, self.prefix))
+ if isinstance(prefix, six.string_types):
+ prefix = utils.Prefix(prefix)
+
+ if prefix.paths[0] in self.prefix.paths:
+ raise RuntimeError("Cannot export library to the same prefix (%s in " \
+ "%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.libraries.values():
k.export(prefix)
diff --git a/beat/backend/python/loader.py b/beat/backend/python/loader.py
index 9b8efbcb707081b3e4129dfea939c35becdd9e39..205e2626f5d1647f29e1da1f6b3cd38cb1bef0ff 100755
--- a/beat/backend/python/loader.py
+++ b/beat/backend/python/loader.py
@@ -34,6 +34,9 @@ import six
import imp
+#----------------------------------------------------------
+
+
def load_module(name, path, uses):
'''Loads the Python file as module, returns a proper Python module
diff --git a/beat/backend/python/test/__init__.py b/beat/backend/python/test/__init__.py
index 5af195d2ab4fda6fa19a56cebdf4a48a7d3646eb..e292e437ae9ac19e5648dd30bcffe0ef95e26beb 100644
--- a/beat/backend/python/test/__init__.py
+++ b/beat/backend/python/test/__init__.py
@@ -30,8 +30,10 @@ import pkg_resources
import tempfile
import sys
+from ..utils import Prefix
-prefix = pkg_resources.resource_filename(__name__, 'prefix')
+
+prefix = Prefix(pkg_resources.resource_filename(__name__, 'prefix'))
if sys.platform == 'darwin':
tmp_prefix = tempfile.mkdtemp(prefix=__name__, suffix='.tmpdir', dir='/tmp')
diff --git a/beat/backend/python/utils.py b/beat/backend/python/utils.py
index 04f096f29d52af7e07a9cc915d43857591210aba..901742bb18a247428db2472f31854ad4fd0af261 100755
--- a/beat/backend/python/utils.py
+++ b/beat/backend/python/utils.py
@@ -37,18 +37,20 @@ import six
from . import hash
+#----------------------------------------------------------
+
def hashed_or_simple(prefix, what, path, suffix='.json'):
"""Returns a hashed path or simple path depending on where the resource is"""
username, right_bit = path.split('/', 1)
hashed_prefix = hash.toUserPath(username)
- candidate = os.path.join(prefix, what, hashed_prefix, right_bit)
+ candidate = prefix.path(os.path.join(what, hashed_prefix, right_bit))
if os.path.exists(candidate + suffix):
return candidate
- return os.path.join(prefix, what, path)
+ return prefix.path(os.path.join(what, path))
#----------------------------------------------------------
@@ -57,7 +59,8 @@ def hashed_or_simple(prefix, what, path, suffix='.json'):
def safe_rmfile(f):
"""Safely removes a file from the disk"""
- if os.path.exists(f): os.unlink(f)
+ if os.path.exists(f):
+ os.unlink(f)
#----------------------------------------------------------
@@ -67,8 +70,12 @@ def safe_rmdir(f):
"""Safely removes the directory containg a given file from the disk"""
d = os.path.dirname(f)
- if not os.path.exists(d): return
- if not os.listdir(d): os.rmdir(d)
+
+ if not os.path.exists(d):
+ return
+
+ if not os.listdir(d):
+ os.rmdir(d)
#----------------------------------------------------------
@@ -97,12 +104,37 @@ def extension_for_language(language):
"""
return dict(
- unknown = '',
- cxx = '.so',
- matlab = '.m',
- python = '.py',
- r = '.r',
- )[language]
+ unknown = '',
+ cxx = '.so',
+ matlab = '.m',
+ python = '.py',
+ r = '.r',
+ )[language]
+
+
+#----------------------------------------------------------
+
+
+class Prefix(object):
+
+ def __init__(self, paths=None):
+ if isinstance(paths, list):
+ self.paths = paths
+ elif paths is not None:
+ self.paths = [paths]
+ else:
+ self.paths = []
+
+ def add(self, path):
+ self.paths.append(path)
+
+ def path(self, filename):
+ for p in self.paths:
+ fullpath = os.path.join(p, filename)
+ if os.path.exists(fullpath):
+ return fullpath
+
+ return os.path.join(self.paths[0], filename)
#----------------------------------------------------------
@@ -126,7 +158,8 @@ class File(object):
def load(self):
mode = 'rb' if self.binary else 'rt'
- with open(self.path, mode) as f: return f.read()
+ with open(self.path, mode) as f:
+ return f.read()
def try_load(self):
@@ -138,21 +171,28 @@ class File(object):
def backup(self):
- if not os.path.exists(self.path): return #no point in backing-up
+ if not os.path.exists(self.path):
+ return #no point in backing-up
+
backup = self.path + '~'
- if os.path.exists(backup): os.remove(backup)
+ if os.path.exists(backup):
+ os.remove(backup)
+
shutil.copy(self.path, backup)
def save(self, contents):
d = os.path.dirname(self.path)
- if not os.path.exists(d): os.makedirs(d)
+ if not os.path.exists(d):
+ os.makedirs(d)
- if os.path.exists(self.path): self.backup()
+ if os.path.exists(self.path):
+ self.backup()
mode = 'wb' if self.binary else 'wt'
- with open(self.path, mode) as f: f.write(contents)
+ with open(self.path, mode) as f:
+ f.write(contents)
def remove(self):
@@ -166,16 +206,7 @@ class File(object):
class Storage(object):
- """Resolves paths for objects that provide only a description
-
- Parameters:
-
- prefix (str): Establishes the prefix of your installation.
-
- name (str): The name of the database object in the format
- ``<name>/<version>``.
-
- """
+ """Resolves paths for objects that provide only a description"""
def __init__(self, path):
@@ -217,11 +248,6 @@ class CodeStorage(object):
Parameters:
- prefix (str): Establishes the prefix of your installation.
-
- name (str): The name of the database object in the format
- ``<name>/<version>``.
-
language (str): One of the valdid programming languages
"""
@@ -262,11 +288,11 @@ class CodeStorage(object):
return hash.hash(dict(
json=hash.hashJSONFile(self.json.path, 'description'),
code=hash.hashFileContents(self.code.path),
- ))
+ ))
else:
return hash.hash(dict(
json=hash.hashJSONFile(self.json.path, 'description'),
- ))
+ ))
def exists(self):
"""If the database declaration file exists"""