Commit 73fcb84a authored by Philip ABBET's avatar Philip ABBET

[unittests] Add tests for the new algorithm APIs

parent 3fff4381
......@@ -90,15 +90,10 @@ class Runner(object):
exception from the user code. Read the documention of :py:func:`run`
for more details.
*args: Constructor parameters for the algorithm (normally none)
**kwargs: Constructor parameters for the algorithm (normally none)
'''
def __init__(self, module, obj_name, algorithm, exc=None, *args,
**kwargs):
def __init__(self, module, obj_name, algorithm, exc=None):
try:
class_ = getattr(module, obj_name)
......@@ -107,15 +102,15 @@ class Runner(object):
type, value, traceback = sys.exc_info()
six.reraise(exc, exc(value), traceback)
else:
raise #just re-raise the user exception
raise # Just re-raise the user exception
self.obj = loader.run(class_, '__new__', exc, *args, **kwargs)
self.obj = loader.run(class_, '__new__', exc)
self.name = module.__name__
self.algorithm = algorithm
self.exc = exc
# if the algorithm does not have a 'setup' method, it is ready by default
self.ready = not hasattr(self.obj, 'setup')
self.prepared = (self.algorithm.api_version == 1) or not hasattr(self.obj, 'prepare')
def _check_parameters(self, parameters):
......@@ -125,7 +120,7 @@ class Runner(object):
algo_parameters = self.algorithm.parameters or {}
valid_keys = set(algo_parameters.keys())
# checks the user is not trying to set an undeclared parameter
# Checks the user is not trying to set an undeclared parameter
if not user_keys.issubset(valid_keys):
err_keys = user_keys - valid_keys
message = "parameters `%s' are not declared for algorithm `%s' - " \
......@@ -150,51 +145,97 @@ class Runner(object):
exc = self.exc or ValueError
raise exc(message)
else: #user has not set a value, use the default
else: # User has not set a value, use the default
value = algo_parameters[key]['default']
# in the end, set the value on the dictionary to be returned
# In the end, set the value on the dictionary to be returned
retval[key] = value
return retval
def setup(self, parameters, *args, **kwargs):
def setup(self, parameters):
'''Sets up the algorithm, only effective on the first call'''
# Only effective on the first call
if self.ready:
return self.ready
completed_parameters = self._check_parameters(parameters)
kwargs['parameters'] = completed_parameters
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
self.ready = loader.run(self.obj, 'setup', self.exc, completed_parameters)
return self.ready
def prepare(self, parameters, *args, **kwargs):
def prepare(self, data_loaders):
'''Let the algorithm process the data on the non-principal channels'''
# Only effective on the first call
if self.prepared:
return self.prepared
# setup() must have run
if not self.ready:
message = "algorithm `%s' is not yet setup" % (self.name,)
exc = self.exc or RuntimeError
raise self.exc(message)
raise exc("Algorithm '%s' is not yet setup" % self.name)
if not hasattr(self.obj, 'prepare'):
# Not available in API version 1
if self.algorithm.api_version == 1:
self.prepared = True
return True
return loader.run(self.obj, 'prepare', self.exc, *args, **kwargs)
# The method is optional
if hasattr(self.obj, 'prepare'):
self.prepared = loader.run(self.obj, 'prepare', self.exc, data_loaders)
else:
self.prepared = True
return self.prepared
def process(self, *args, **kwargs):
def process(self, inputs=None, data_loaders=None, outputs=None, output=None):
'''Runs through data'''
exc = self.exc or RuntimeError
def _check_argument(argument, name):
if argument is None:
raise exc('Missing argument: %s' % name)
# setup() must have run
if not self.ready:
message = "algorithm `%s' is not yet setup" % (self.name,)
exc = self.exc or RuntimeError
raise self.exc(message)
raise exc("Algorithm '%s' is not yet setup" % self.name)
return loader.run(self.obj, 'process', self.exc, *args, **kwargs)
# prepare() must have run
if not self.prepared:
raise exc("Algorithm '%s' is not yet prepared" % self.name)
# Call the correct version of process()
if self.algorithm.isAnalyzer:
_check_argument(output, 'output')
outputs_to_use = output
else:
_check_argument(outputs, 'outputs')
outputs_to_use = outputs
if self.algorithm.type == Algorithm.LEGACY:
_check_argument(inputs, 'inputs')
return loader.run(self.obj, 'process', self.exc, inputs, outputs_to_use)
else:
_check_argument(data_loaders, 'data_loaders')
if self.algorithm.type == Algorithm.SEQUENTIAL:
_check_argument(inputs, 'inputs')
return loader.run(self.obj, 'process', self.exc, inputs, data_loaders,
outputs_to_use)
elif self.algorithm.type == Algorithm.AUTONOMOUS:
return loader.run(self.obj, 'process', self.exc, data_loaders,
outputs_to_use)
else:
raise exc('Unknown algorithm type: %s' % self.algorithm.type)
def __getattr__(self, key):
......@@ -568,6 +609,11 @@ class Algorithm(object):
return value
@property
def isAnalyzer(self):
return (self.results is not None)
@property
def results(self):
return self.data.get('results')
......@@ -601,6 +647,67 @@ class Algorithm(object):
return value
@property
def description(self):
"""The short description for this object"""
return self.data.get('description', None)
@description.setter
def description(self, value):
"""Sets the short description for this object"""
self.data['description'] = value
@property
def documentation(self):
"""The full-length description for this object"""
if not self._name:
raise RuntimeError("algorithm has no name")
if self.storage.doc.exists():
return self.storage.doc.load()
return None
@documentation.setter
def documentation(self, value):
"""Sets the full-length description for this object"""
if not self._name:
raise RuntimeError("algorithm has no name")
if hasattr(value, 'read'):
self.storage.doc.save(value.read())
else:
self.storage.doc.save(value)
def hash(self):
"""Returns the hexadecimal hash for the current algorithm"""
if not self._name:
raise RuntimeError("algorithm has no name")
return self.storage.hash()
def result_dataformat(self):
"""Generates, on-the-fly, the dataformat for the result readout"""
if not self.results:
raise TypeError("algorithm `%s' is a block algorithm, not an analyzer" \
% (self.name))
format = dataformat.DataFormat(self.prefix,
dict([(k, v['type']) for k,v in self.results.items()]))
format.name = 'analysis:' + self.name
return format
def uses_dict(self):
"""Returns the usage dictionary for all dependent modules"""
......@@ -665,64 +772,3 @@ class Algorithm(object):
raise #just re-raise the user exception
return Runner(self.__module, klass, self, exc)
@property
def description(self):
"""The short description for this object"""
return self.data.get('description', None)
@description.setter
def description(self, value):
"""Sets the short description for this object"""
self.data['description'] = value
@property
def documentation(self):
"""The full-length description for this object"""
if not self._name:
raise RuntimeError("algorithm has no name")
if self.storage.doc.exists():
return self.storage.doc.load()
return None
@documentation.setter
def documentation(self, value):
"""Sets the full-length description for this object"""
if not self._name:
raise RuntimeError("algorithm has no name")
if hasattr(value, 'read'):
self.storage.doc.save(value.read())
else:
self.storage.doc.save(value)
def hash(self):
"""Returns the hexadecimal hash for the current algorithm"""
if not self._name:
raise RuntimeError("algorithm has no name")
return self.storage.hash()
def result_dataformat(self):
"""Generates, on-the-fly, the dataformat for the result readout"""
if not self.results:
raise TypeError("algorithm `%s' is a block algorithm, not an analyzer" \
% (self.name))
format = dataformat.DataFormat(self.prefix,
dict([(k, v['type']) for k,v in self.results.items()]))
format.name = 'analysis:' + self.name
return format
......@@ -323,8 +323,8 @@ class DataLoaderList(object):
if isinstance(name_or_index, six.string_types):
return [x for x in self._loaders if x.channel == name_or_index][0]
elif isinstance(index, int):
return self._loaders[index]
elif isinstance(name_or_index, int):
return self._loaders[name_or_index]
except:
pass
......@@ -345,3 +345,12 @@ class DataLoaderList(object):
return [ k for k in self._loaders if input_name in k.input_names() ][0]
except:
return None
def secondaries(self):
secondaries_list = DataLoaderList()
for data_loader in self._loaders:
if data_loader is not self.main_loader:
secondaries_list.add(data_loader)
return secondaries_list
......@@ -69,6 +69,9 @@ def load_module(name, path, uses):
return retval
#----------------------------------------------------------
def run(obj, method, exc=None, *args, **kwargs):
'''Runs a method on the object and protects its execution
......@@ -96,7 +99,9 @@ def run(obj, method, exc=None, *args, **kwargs):
'''
try:
if method == '__new__': return obj(*args, **kwargs)
if method == '__new__':
return obj(*args, **kwargs)
return getattr(obj, method)(*args, **kwargs)
except Exception as e:
if exc is not None:
......
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"parameters": {
"sync": {
"default": "in1",
"type": "string"
}
},
"groups": [
{
"inputs": {
"in1": {
"type": "user/single_integer/1"
},
"in2": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
}
]
}
\ No newline at end of file
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
self.sync = parameters['sync']
return True
def process(self, data_loaders, outputs):
data_loader = data_loaders.loaderOf('in1')
for i in range(data_loader.count(self.sync)):
view = data_loader.view(self.sync, i)
(data, start, end) = view[view.count() - 1]
outputs['out'].write({
'value': data['in1'].value + data['in2'].value,
},
end
)
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"in1": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
},
{
"inputs": {
"in2": {
"type": "user/single_integer/1"
}
}
}
]
}
\ No newline at end of file
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def prepare(self, data_loaders):
data_loader = data_loaders.loaderOf('in2')
(data, start, end) = data_loader[0]
self.offset = data['in2'].value
return True
def process(self, data_loaders, outputs):
data_loader = data_loaders.loaderOf('in1')
for i in range(data_loader.count()):
(data, start, end) = data_loader[i]
outputs['out'].write({
'value': data['in1'].value + self.offset,
},
end
)
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"value": {
"type": "user/single_integer/1"
},
"offset_index": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
},
{
"inputs": {
"offset": {
"type": "user/single_integer/1"
}
}
}
]
}
\ No newline at end of file
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def process(self, data_loaders, outputs):
values = data_loaders.loaderOf('value')
offsets = data_loaders.loaderOf('offset')
for i in range(values.count()):
(data, start, end) = values[i]
offset_index = data['offset_index'].value
(data2, start2, end2) = offsets[offset_index]
outputs['out'].write({
'value': data['value'].value + data2['offset'].value,
},
end
)
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"in": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def process(self, data_loaders, outputs):
data_loader = data_loaders.loaderOf('in')
for i in range(data_loader.count()):
(data, start, end) = data_loader[i]
outputs['out'].write({
'value': data['in'].value,
},
end
)
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"