Commit 07818995 authored by Philip ABBET's avatar Philip ABBET
Browse files

Force 4-spaces indentation

parent de1bbd93
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -42,337 +42,337 @@ from .dataformat import DataFormat
class Storage(utils.CodeStorage):
"""Resolves paths for databases
"""Resolves paths for databases
Parameters:
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
name (str): The name of the database object in the format
``<name>/<version>``.
"""
"""
def __init__(self, prefix, name):
def __init__(self, prefix, name):
if name.count('/') != 1:
raise RuntimeError("invalid database name: `%s'" % name)
if name.count('/') != 1:
raise RuntimeError("invalid database name: `%s'" % name)
self.name, self.version = name.split('/')
self.fullname = name
self.name, self.version = name.split('/')
self.fullname = name
path = os.path.join(prefix, 'databases', name)
super(Storage, self).__init__(path, 'python') #views are coded in Python
path = os.path.join(prefix, 'databases', name)
super(Storage, self).__init__(path, 'python') #views are coded in Python
class View(object):
'''A special loader class for database views, with specialized methods
'''A special loader class for database views, with specialized methods
Parameters:
Parameters:
db_name (str): The full name of the database object for this view
db_name (str): The full name of the database object for this view
module (module): The preloaded module containing the database views as
returned by :py:func:`beat.core.loader.load_module`.
module (module): The preloaded module containing the database views as
returned by :py:func:`beat.core.loader.load_module`.
prefix (str, path): The prefix path for the current installation
prefix (str, path): The prefix path for the current installation
root_folder (str, path): The path pointing to the root folder of this
database
root_folder (str, path): The path pointing to the root folder of this
database
exc (class): The class to use as base exception when translating the
exception from the user code. Read the documention of :py:func:`run`
for more details.
exc (class): The class to use as base exception when translating the
exception from the user code. Read the documention of :py:func:`run`
for more details.
*args: Constructor parameters for the database view. Normally, none.
*args: Constructor parameters for the database view. Normally, none.
**kwargs: Constructor parameters for the database view. Normally, none.
**kwargs: Constructor parameters for the database view. Normally, none.
'''
'''
def __init__(self, module, definition, prefix, root_folder, exc=None,
*args, **kwargs):
def __init__(self, module, definition, prefix, root_folder, exc=None,
*args, **kwargs):
try:
class_ = getattr(module, definition['view'])
except Exception as e:
if exc is not None:
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
else:
raise #just re-raise the user exception
try:
class_ = getattr(module, definition['view'])
except Exception as e:
if exc is not None:
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
else:
raise #just re-raise the user exception
self.obj = loader.run(class_, '__new__', exc, *args, **kwargs)
self.ready = False
self.prefix = prefix
self.root_folder = root_folder
self.definition = definition
self.exc = exc or RuntimeError
self.outputs = None
self.obj = loader.run(class_, '__new__', exc, *args, **kwargs)
self.ready = False
self.prefix = prefix
self.root_folder = root_folder
self.definition = definition
self.exc = exc or RuntimeError
self.outputs = None
def prepare_outputs(self):
'''Prepares the outputs of the dataset'''
def prepare_outputs(self):
'''Prepares the outputs of the dataset'''
from .outputs import Output, OutputList
from .data import MemoryDataSink
from .dataformat import DataFormat
from .outputs import Output, OutputList
from .data import MemoryDataSink
from .dataformat import DataFormat
# create the stock outputs for this dataset, so data is dumped
# on a in-memory sink
self.outputs = OutputList()
for out_name, out_format in self.definition.get('outputs', {}).items():
data_sink = MemoryDataSink()
data_sink.dataformat = DataFormat(self.prefix, out_format)
data_sink.setup([])
self.outputs.add(Output(out_name, data_sink, dataset_output=True))
# create the stock outputs for this dataset, so data is dumped
# on a in-memory sink
self.outputs = OutputList()
for out_name, out_format in self.definition.get('outputs', {}).items():
data_sink = MemoryDataSink()
data_sink.dataformat = DataFormat(self.prefix, out_format)
data_sink.setup([])
self.outputs.add(Output(out_name, data_sink, dataset_output=True))
def setup(self, *args, **kwargs):
'''Sets up the view'''
def setup(self, *args, **kwargs):
'''Sets up the view'''
kwargs.setdefault('root_folder', self.root_folder)
kwargs.setdefault('parameters', self.definition.get('parameters', {}))
kwargs.setdefault('root_folder', self.root_folder)
kwargs.setdefault('parameters', self.definition.get('parameters', {}))
if 'outputs' not in kwargs:
kwargs['outputs'] = self.outputs
else:
self.outputs = kwargs['outputs'] #record outputs nevertheless
if 'outputs' not in kwargs:
kwargs['outputs'] = self.outputs
else:
self.outputs = kwargs['outputs'] #record outputs nevertheless
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
if not self.ready:
raise self.exc("unknow setup failure")
if not self.ready:
raise self.exc("unknow setup failure")
return self.ready
return self.ready
def input_group(self, name='default', exclude_outputs=[]):
'''A memory-source input group matching the outputs from the view'''
def input_group(self, name='default', exclude_outputs=[]):
'''A memory-source input group matching the outputs from the view'''
if not self.ready:
raise self.exc("database view not yet setup")
if not self.ready:
raise self.exc("database view not yet setup")
from .data import MemoryDataSource
from .outputs import SynchronizationListener
from .inputs import Input, InputGroup
from .data import MemoryDataSource
from .outputs import SynchronizationListener
from .inputs import Input, InputGroup
# Setup the inputs
synchronization_listener = SynchronizationListener()
input_group = InputGroup(name,
synchronization_listener=synchronization_listener,
restricted_access=False)
# Setup the inputs
synchronization_listener = SynchronizationListener()
input_group = InputGroup(name,
synchronization_listener=synchronization_listener,
restricted_access=False)
for output in self.outputs:
if output.name in exclude_outputs: continue
data_source = MemoryDataSource(self.done, next_callback=self.next)
output.data_sink.data_sources.append(data_source)
input_group.add(Input(output.name,
output.data_sink.dataformat, data_source))
for output in self.outputs:
if output.name in exclude_outputs: continue
data_source = MemoryDataSource(self.done, next_callback=self.next)
output.data_sink.data_sources.append(data_source)
input_group.add(Input(output.name,
output.data_sink.dataformat, data_source))
return input_group
return input_group
def done(self, *args, **kwargs):
'''Checks if the view is done'''
def done(self, *args, **kwargs):
'''Checks if the view is done'''
if not self.ready:
raise self.exc("database view not yet setup")
if not self.ready:
raise self.exc("database view not yet setup")
return loader.run(self.obj, 'done', self.exc, *args, **kwargs)
return loader.run(self.obj, 'done', self.exc, *args, **kwargs)
def next(self, *args, **kwargs):
'''Runs through the next data chunk'''
def next(self, *args, **kwargs):
'''Runs through the next data chunk'''
if not self.ready:
raise self.exc("database view not yet setup")
return loader.run(self.obj, 'next', self.exc, *args, **kwargs)
if not self.ready:
raise self.exc("database view not yet setup")
return loader.run(self.obj, 'next', self.exc, *args, **kwargs)
def __getattr__(self, key):
'''Returns an attribute of the view - only called at last resort'''
return getattr(self.obj, key)
def __getattr__(self, key):
'''Returns an attribute of the view - only called at last resort'''
return getattr(self.obj, key)
class Database(object):
"""Databases define the start point of the dataflow in an experiment.
"""Databases define the start point of the dataflow in an experiment.
Parameters:
Parameters:
prefix (str): Establishes the prefix of your installation.
prefix (str): Establishes the prefix of your installation.
name (str): The fully qualified database name (e.g. ``db/1``)
name (str): The fully qualified database name (e.g. ``db/1``)
dataformat_cache (dict, optional): A dictionary mapping dataformat names
to loaded dataformats. This parameter is optional and, if passed, may
greatly speed-up database loading times as dataformats that are already
loaded may be re-used. If you use this parameter, you must guarantee
that the cache is refreshed as appropriate in case the underlying
dataformats change.
dataformat_cache (dict, optional): A dictionary mapping dataformat names
to loaded dataformats. This parameter is optional and, if passed, may
greatly speed-up database loading times as dataformats that are already
loaded may be re-used. If you use this parameter, you must guarantee
that the cache is refreshed as appropriate in case the underlying
dataformats change.
Attributes:
Attributes:
name (str): The full, valid name of this database
name (str): The full, valid name of this database
data (dict): The original data for this database, as loaded by our JSON
decoder.
data (dict): The original data for this database, as loaded by our JSON
decoder.
"""
"""
def __init__(self, prefix, name, dataformat_cache=None):
def __init__(self, prefix, name, dataformat_cache=None):
self._name = None
self.prefix = prefix
self.dataformats = {} # preloaded dataformats
self.storage = None
self._name = None
self.prefix = prefix
self.dataformats = {} # preloaded dataformats
self.storage = None
self.errors = []
self.data = None
self.errors = []
self.data = None
# if the user has not provided a cache, still use one for performance
dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
# if the user has not provided a cache, still use one for performance
dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
self._load(name, dataformat_cache)
self._load(name, dataformat_cache)
def _load(self, data, dataformat_cache):
"""Loads the database"""
def _load(self, data, dataformat_cache):
"""Loads the database"""
self._name = data
self._name = data
self.storage = Storage(self.prefix, self._name)
json_path = self.storage.json.path
if not self.storage.json.exists():
self.errors.append('Database declaration file not found: %s' % json_path)
return
self.storage = Storage(self.prefix, self._name)
json_path = self.storage.json.path
if not self.storage.json.exists():
self.errors.append('Database declaration file not found: %s' % json_path)
return
with open(json_path, 'rb') as f:
self.data = simplejson.load(f)
with open(json_path, 'rb') as f:
self.data = simplejson.load(f)
for protocol in self.data['protocols']:
for _set in protocol['sets']:
for protocol in self.data['protocols']:
for _set in protocol['sets']:
for key, value in _set['outputs'].items():
for key, value in _set['outputs'].items():
if value in self.dataformats:
continue
if value in self.dataformats:
continue
if value in dataformat_cache:
dataformat = dataformat_cache[value]
else:
dataformat = DataFormat(self.prefix, value)
dataformat_cache[value] = dataformat
if value in dataformat_cache:
dataformat = dataformat_cache[value]
else:
dataformat = DataFormat(self.prefix, value)
dataformat_cache[value] = dataformat
self.dataformats[value] = dataformat
self.dataformats[value] = dataformat
@property
def name(self):
"""Returns the name of this object
"""
return self._name or '__unnamed_database__'
@property
def name(self):
"""Returns the name of this object
"""
return self._name or '__unnamed_database__'
@property
def schema_version(self):
"""Returns the schema version"""
return self.data.get('schema_version', 1)
@property
def schema_version(self):
"""Returns the schema version"""
return self.data.get('schema_version', 1)
@property
def valid(self):
return not bool(self.errors)
@property
def valid(self):
return not bool(self.errors)
@property
def protocols(self):
"""The declaration of all the protocols of the database"""
@property
def protocols(self):
"""The declaration of all the protocols of the database"""
data = self.data['protocols']
return dict(zip([k['name'] for k in data], data))
data = self.data['protocols']
return dict(zip([k['name'] for k in data], data))
def protocol(self, name):
"""The declaration of a specific protocol in the database"""
def protocol(self, name):
"""The declaration of a specific protocol in the database"""
return self.protocols[name]
return self.protocols[name]
@property
def protocol_names(self):
"""Names of protocols declared for this database"""
@property
def protocol_names(self):
"""Names of protocols declared for this database"""
data = self.data['protocols']
return [k['name'] for k in data]
data = self.data['protocols']
return [k['name'] for k in data]
def sets(self, protocol):
"""The declaration of a specific set in the database protocol"""
def sets(self, protocol):
"""The declaration of a specific set in the database protocol"""
data = self.protocol(protocol)['sets']
return dict(zip([k['name'] for k in data], data))
data = self.protocol(protocol)['sets']
return dict(zip([k['name'] for k in data], data))
def set(self, protocol, name):
"""The declaration of all the protocols of the database"""
def set(self, protocol, name):
"""The declaration of all the protocols of the database"""
return self.sets(protocol)[name]
return self.sets(protocol)[name]
def set_names(self, protocol):
"""The names of sets in a given protocol for this database"""
def set_names(self, protocol):
"""The names of sets in a given protocol for this database"""
data = self.protocol(protocol)['sets']
return [k['name'] for k in data]
data = self.protocol(protocol)['sets']
return [k['name'] for k in data]
def view(self, protocol, name, exc=None):
"""Returns the database view, given the protocol and the set name
def view(self, protocol, name, exc=None):
"""Returns the database view, given the protocol and the set name
Parameters:
Parameters:
protocol (str): The name of the protocol where to retrieve the view from
protocol (str): The name of the protocol where to retrieve the view from
name (str): The name of the set in the protocol where to retrieve the
view from
name (str): The name of the set in the protocol where to retrieve the
view from
exc (class): If passed, must be a valid exception class that will be
used to report errors in the read-out of this database's view.
exc (class): If passed, must be a valid exception class that will be
used to report errors in the read-out of this database's view.
Returns:
Returns:
The database view, which will be constructed, but not setup. You
**must** set it up before using methods ``done`` or ``next``.
The database view, which will be constructed, but not setup. You
**must** set it up before using methods ``done`` or ``next``.
"""
"""
if not self._name:
exc = exc or RuntimeError
raise exc("database has no name")
if not self.valid:
message = "cannot load view for set `%s' of protocol `%s' " \
"from invalid database (%s)" % (protocol, name, self.name)
if exc: raise exc(message)
raise RuntimeError(message)
# loads the module only once through the lifetime of the database object
try:
if not hasattr(self, '_module'):
self._module = loader.load_module(self.name.replace(os.sep, '_'),
self.storage.code.path, {})
except Exception as e:
if exc is not None:
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
else:
raise #just re-raise the user exception
if not self._name:
exc = exc or RuntimeError
raise exc("database has no name")
if not self.valid:
message = "cannot load view for set `%s' of protocol `%s' " \
"from invalid database (%s)" % (protocol, name, self.name)
if exc: raise exc(message)
raise RuntimeError(message)
# loads the module only once through the lifetime of the database object
try:
if not hasattr(self, '_module'):
self._module = loader.load_module(self.name.replace(os.sep, '_'),
self.storage.code.path, {})
except Exception as e:
if exc is not None:
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
else:
raise #just re-raise the user exception
return View(self._module, self.set(protocol, name), self.prefix,
self.data['root_folder'], exc)
return View(self._module, self.set(protocol, name), self.prefix,
self.data['root_folder'], exc)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -38,82 +38,82 @@ import os
def _sha256(s):
"""A python2/3 replacement for :py:func:`haslib.sha256`"""
"""A python2/3 replacement for :py:func:`haslib.sha256`"""
try:
if isinstance(s, str): s = six.u(s)
return hashlib.sha256(s.encode('utf8')).hexdigest()
except:
return hashlib.sha256(s).hexdigest()
try:
if isinstance(s, str): s = six.u(s)
return hashlib.sha256(s.encode('utf8')).hexdigest()
except:
return hashlib.sha256(s).hexdigest()
def _stringify(dictionary):
names = sorted(dictionary.keys())
names = sorted(dictionary.keys())
converted_dictionary = '{'
for name in names:
converted_dictionary += '"%s":%s,' % (name, str(dictionary[name]))
converted_dictionary = '{'
for name in names: