Commit 3aaf8666 authored by Philip ABBET's avatar Philip ABBET

Add a 'Prefix' class, to support loading from several folders

parent df468c7b
......@@ -49,7 +49,8 @@ class Storage(utils.CodeStorage):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
......@@ -261,7 +262,8 @@ class Algorithm(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of your
installation.
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
......@@ -850,9 +852,12 @@ class Algorithm(object):
if not self.valid:
raise RuntimeError("algorithm is not valid")
if os.path.samefile(prefix, self.prefix):
raise RuntimeError("Cannot export algorithm to the same prefix (%s == " \
"%s)" % (prefix, self.prefix))
if isinstance(prefix, six.string_types):
prefix = utils.Prefix(prefix)
if prefix.paths[0] in self.prefix.paths:
raise RuntimeError("Cannot export algorithm to the same prefix (%s in " \
"%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.libraries.values():
k.export(prefix)
......
......@@ -305,7 +305,8 @@ class CachedDataSource(DataSource):
filename (str): Name of the file to read the data from
prefix (str, path): Path to the prefix where the dataformats are stored.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
start_index (int): The starting index (if not set or set to
``None``, the default, read data from the begin of file)
......@@ -489,7 +490,8 @@ class DatabaseOutputDataSource(DataSource):
Parameters:
prefix (str, path): Path to the prefix where the dataformats are stored.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
start_index (int): The starting index (if not set or set to
``None``, the default, read data from the begin of file)
......@@ -609,7 +611,8 @@ class RemoteDataSource(DataSource):
dataformat_name (str): Name of the data format.
prefix (str, path): Path to the prefix where the dataformats are stored.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
unpack (bool): Indicates if the data must be unpacked or not
......@@ -976,12 +979,12 @@ class CachedDataSink(DataSink):
#----------------------------------------------------------
def load_data_index(cache_prefix, hash_path):
def load_data_index(cache_root, hash_path):
"""Loads a cached-data index if it exists. Returns empty otherwise.
Parameters:
cache_prefix (str): The path to the root of the cache directory
cache_root (str): The path to the root of the cache directory
hash_path (str): The hashed path of the input you wish to load the indexes
for, as it is returned by the utility function
......@@ -996,7 +999,7 @@ def load_data_index(cache_prefix, hash_path):
"""
name_no_extension = os.path.splitext(hash_path)[0] # remove .data
index_stem = os.path.join(cache_prefix, name_no_extension)
index_stem = os.path.join(cache_root, name_no_extension)
index_glob = index_stem + '*.index'
candidates = glob.glob(index_glob)
......
......@@ -52,7 +52,8 @@ class Storage(utils.CodeStorage):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
......@@ -67,7 +68,7 @@ class Storage(utils.CodeStorage):
self.name, self.version = name.split('/')
self.fullname = name
path = os.path.join(prefix, 'databases', name)
path = prefix.path(os.path.join('databases', name))
super(Storage, self).__init__(path, 'python') #views are coded in Python
......@@ -84,7 +85,8 @@ class Runner(object):
module (module): The preloaded module containing the database views as
returned by :py:func:`beat.core.loader.load_module`.
prefix (str, path): The prefix path for the current installation
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
root_folder (str, path): The path pointing to the root folder of this
database
......@@ -177,11 +179,6 @@ class Runner(object):
return loader.run(self.obj, 'get', self.exc, output, index)
def __getattr__(self, key):
'''Returns an attribute of the view - only called at last resort'''
return getattr(self.obj, key)
def objects(self):
return self.obj.objs
......@@ -195,7 +192,8 @@ class Database(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The fully qualified database name (e.g. ``db/1``)
......
......@@ -40,13 +40,16 @@ from . import utils
from .baseformat import baseformat
#----------------------------------------------------------
class Storage(utils.Storage):
"""Resolves paths for dataformats
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The name of the dataformat object in the format
``<user>/<name>/<version>``.
......@@ -70,13 +73,16 @@ class Storage(utils.Storage):
return super(Storage, self).hash('#description')
#----------------------------------------------------------
class DataFormat(object):
"""Data formats define the chunks of data that circulate between blocks.
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
data (str, dict): The fully qualified algorithm name (e.g. ``user/algo/1``)
or a dictionary representing the data format (for analyzer results).
......@@ -480,7 +486,8 @@ class DataFormat(object):
Parameters:
prefix (str): A path to a prefix that must different then my own.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
Returns:
......@@ -500,9 +507,12 @@ class DataFormat(object):
if not self.valid:
raise RuntimeError("dataformat is not valid")
if os.path.samefile(prefix, self.prefix):
raise RuntimeError("Cannot dataformat object to the same prefix (%s " \
"== %s)" % (prefix, self.prefix))
if isinstance(prefix, six.string_types):
prefix = utils.Prefix(prefix)
if prefix.paths[0] in self.prefix.paths:
raise RuntimeError("Cannot export dataformat to the same prefix (%s in " \
"%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.referenced.values():
k.export(prefix)
......
......@@ -46,7 +46,8 @@ class DBExecutor(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
data (dict, str): The piece of data representing the block to be executed.
It must validate against the schema defined for execution blocks. If a
......
......@@ -41,6 +41,7 @@ from .algorithm import Algorithm
from .helpers import create_inputs_from_configuration
from .helpers import create_outputs_from_configuration
from .helpers import AccessMode
from .utils import Prefix
from . import stats
......@@ -85,7 +86,7 @@ class Executor(object):
with open(self.configuration, 'rb') as f:
self.data = simplejson.load(f)
self.prefix = os.path.join(directory, 'prefix')
self.prefix = Prefix(os.path.join(directory, 'prefix'))
self.runner = None
# Temporary caches, if the user has not set them, for performance
......
......@@ -36,13 +36,16 @@ from . import loader
from . import utils
#----------------------------------------------------------
class Storage(utils.CodeStorage):
"""Resolves paths for libraries
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The name of the library object in the format
``<user>/<name>/<version>``.
......@@ -62,6 +65,8 @@ class Storage(utils.CodeStorage):
super(Storage, self).__init__(path, language)
#----------------------------------------------------------
class Library(object):
"""Librarys represent independent algorithm components within the platform.
......@@ -73,7 +78,8 @@ class Library(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
......@@ -346,7 +352,8 @@ class Library(object):
Parameters:
prefix (str): A path to a prefix that must different then my own.
prefix (beat.backend.python.utils.Prefix): Establishes the prefix of
your installation.
Returns:
......@@ -366,9 +373,12 @@ class Library(object):
if not self.valid:
raise RuntimeError("library is not valid")
if os.path.samefile(prefix, self.prefix):
raise RuntimeError("Cannot library object to the same prefix (%s == " \
"%s)" % (prefix, self.prefix))
if isinstance(prefix, six.string_types):
prefix = utils.Prefix(prefix)
if prefix.paths[0] in self.prefix.paths:
raise RuntimeError("Cannot export library to the same prefix (%s in " \
"%s)" % (prefix.paths[0], self.prefix.paths))
for k in self.libraries.values():
k.export(prefix)
......
......@@ -34,6 +34,9 @@ import six
import imp
#----------------------------------------------------------
def load_module(name, path, uses):
'''Loads the Python file as module, returns a proper Python module
......
......@@ -30,8 +30,10 @@ import pkg_resources
import tempfile
import sys
from ..utils import Prefix
prefix = pkg_resources.resource_filename(__name__, 'prefix')
prefix = Prefix(pkg_resources.resource_filename(__name__, 'prefix'))
if sys.platform == 'darwin':
tmp_prefix = tempfile.mkdtemp(prefix=__name__, suffix='.tmpdir', dir='/tmp')
......
......@@ -37,18 +37,20 @@ import six
from . import hash
#----------------------------------------------------------
def hashed_or_simple(prefix, what, path, suffix='.json'):
"""Returns a hashed path or simple path depending on where the resource is"""
username, right_bit = path.split('/', 1)
hashed_prefix = hash.toUserPath(username)
candidate = os.path.join(prefix, what, hashed_prefix, right_bit)
candidate = prefix.path(os.path.join(what, hashed_prefix, right_bit))
if os.path.exists(candidate + suffix):
return candidate
return os.path.join(prefix, what, path)
return prefix.path(os.path.join(what, path))
#----------------------------------------------------------
......@@ -57,7 +59,8 @@ def hashed_or_simple(prefix, what, path, suffix='.json'):
def safe_rmfile(f):
"""Safely removes a file from the disk"""
if os.path.exists(f): os.unlink(f)
if os.path.exists(f):
os.unlink(f)
#----------------------------------------------------------
......@@ -67,8 +70,12 @@ def safe_rmdir(f):
"""Safely removes the directory containg a given file from the disk"""
d = os.path.dirname(f)
if not os.path.exists(d): return
if not os.listdir(d): os.rmdir(d)
if not os.path.exists(d):
return
if not os.listdir(d):
os.rmdir(d)
#----------------------------------------------------------
......@@ -97,12 +104,37 @@ def extension_for_language(language):
"""
return dict(
unknown = '',
cxx = '.so',
matlab = '.m',
python = '.py',
r = '.r',
)[language]
unknown = '',
cxx = '.so',
matlab = '.m',
python = '.py',
r = '.r',
)[language]
#----------------------------------------------------------
class Prefix(object):
def __init__(self, paths=None):
if isinstance(paths, list):
self.paths = paths
elif paths is not None:
self.paths = [paths]
else:
self.paths = []
def add(self, path):
self.paths.append(path)
def path(self, filename):
for p in self.paths:
fullpath = os.path.join(p, filename)
if os.path.exists(fullpath):
return fullpath
return os.path.join(self.paths[0], filename)
#----------------------------------------------------------
......@@ -126,7 +158,8 @@ class File(object):
def load(self):
mode = 'rb' if self.binary else 'rt'
with open(self.path, mode) as f: return f.read()
with open(self.path, mode) as f:
return f.read()
def try_load(self):
......@@ -138,21 +171,28 @@ class File(object):
def backup(self):
if not os.path.exists(self.path): return #no point in backing-up
if not os.path.exists(self.path):
return #no point in backing-up
backup = self.path + '~'
if os.path.exists(backup): os.remove(backup)
if os.path.exists(backup):
os.remove(backup)
shutil.copy(self.path, backup)
def save(self, contents):
d = os.path.dirname(self.path)
if not os.path.exists(d): os.makedirs(d)
if not os.path.exists(d):
os.makedirs(d)
if os.path.exists(self.path): self.backup()
if os.path.exists(self.path):
self.backup()
mode = 'wb' if self.binary else 'wt'
with open(self.path, mode) as f: f.write(contents)
with open(self.path, mode) as f:
f.write(contents)
def remove(self):
......@@ -166,16 +206,7 @@ class File(object):
class Storage(object):
"""Resolves paths for objects that provide only a description
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
"""
"""Resolves paths for objects that provide only a description"""
def __init__(self, path):
......@@ -217,11 +248,6 @@ class CodeStorage(object):
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
language (str): One of the valdid programming languages
"""
......@@ -262,11 +288,11 @@ class CodeStorage(object):
return hash.hash(dict(
json=hash.hashJSONFile(self.json.path, 'description'),
code=hash.hashFileContents(self.code.path),
))
))
else:
return hash.hash(dict(
json=hash.hashJSONFile(self.json.path, 'description'),
))
))
def exists(self):
"""If the database declaration file exists"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment