Commit 3fff4381 authored by Philip ABBET's avatar Philip ABBET

New API for the algorithms

parent ec74eae8
......@@ -41,6 +41,8 @@ from . import loader
from . import utils
#----------------------------------------------------------
class Storage(utils.CodeStorage):
"""Resolves paths for algorithms
......@@ -67,6 +69,8 @@ class Storage(utils.CodeStorage):
super(Storage, self).__init__(path, language)
#----------------------------------------------------------
class Runner(object):
'''A special loader class for algorithms, with specialized methods
......@@ -79,13 +83,13 @@ class Runner(object):
obj_name (str): The name of the object within the module you're interested
on
algorithm (object): The algorithm instance that is used for parameter
checking.
exc (class): The class to use as base exception when translating the
exception from the user code. Read the documention of :py:func:`run`
for more details.
algorithm (object): The algorithm instance that is used for parameter
checking.
*args: Constructor parameters for the algorithm (normally none)
**kwargs: Constructor parameters for the algorithm (normally none)
......@@ -133,9 +137,7 @@ class Runner(object):
exc = self.exc or KeyError
raise exc(message)
# checks all values set by the user are in range (if a range is set)
retval = dict() #dictionary with checked user parameters and defaults
retval = dict()
for key, definition in algo_parameters.items():
......@@ -160,17 +162,29 @@ class Runner(object):
def setup(self, parameters, *args, **kwargs):
'''Sets up the algorithm, only effective on the first call'''
if self.ready: return self.ready
if self.ready:
return self.ready
completed_parameters = self._check_parameters(parameters) #may raise
completed_parameters = self._check_parameters(parameters)
kwargs['parameters'] = completed_parameters
if hasattr(self.obj, 'setup'):
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
return self.ready
else:
def prepare(self, parameters, *args, **kwargs):
'''Let the algorithm process the data on the non-principal channels'''
if not self.ready:
message = "algorithm `%s' is not yet setup" % (self.name,)
exc = self.exc or RuntimeError
raise self.exc(message)
if not hasattr(self.obj, 'prepare'):
return True
return loader.run(self.obj, 'prepare', self.exc, *args, **kwargs)
def process(self, *args, **kwargs):
'''Runs through data'''
......@@ -188,6 +202,8 @@ class Runner(object):
return getattr(self.obj, key)
#----------------------------------------------------------
class Algorithm(object):
"""Algorithms represent runnable components within the platform.
......@@ -262,6 +278,11 @@ class Algorithm(object):
"""
LEGACY = 'legacy'
SEQUENTIAL = 'sequential'
AUTONOMOUS = 'autonomous'
def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
self._name = None
......@@ -451,6 +472,21 @@ class Algorithm(object):
return self.data['language']
@property
def api_version(self):
"""Returns the API version"""
return self.data.get('api_version', 1)
@property
def type(self):
"""Returns the type of algorithm"""
if self.api_version == 1:
return Algorithm.LEGACY
return self.data.get('type', Algorithm.SEQUENTIAL)
@language.setter
def language(self, value):
"""Sets the current executable code programming language"""
......
This diff is collapsed.
......@@ -495,147 +495,6 @@ class InputGroup:
#----------------------------------------------------------
class DataView(object):
def __init__(self, data_loader_group, data_indices):
self.infos = {}
self.data_indices = data_indices
self.nb_data_units = len(data_indices)
self.data_index = data_indices[0][0]
self.data_index_end = data_indices[-1][1]
for input_name, infos in data_loader_group.infos.items():
input_data_indices = []
current_start = self.data_index
for i in range(self.data_index, self.data_index_end + 1):
for indices in infos['data_indices']:
if indices[1] == i:
input_data_indices.append( (current_start, i) )
current_start = i + 1
break
if (len(input_data_indices) == 0) or (input_data_indices[-1][1] != self.data_index_end):
input_data_indices.append( (current_start, self.data_index_end) )
self.infos[input_name] = dict(
cached_file = infos['cached_file'],
data_indices = input_data_indices,
data = None,
start_index = -1,
end_index = -1,
)
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return None
else:
return self.nb_data_units
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class DataLoaderGroup(object):
def __init__(self, channel):
self.channel = str(channel)
self.infos = {}
self.mixed_data_indices = None
self.nb_data_units = 0
self.data_index = -1 # Lower index across all inputs
self.data_index_end = -1 # Bigger index across all inputs
def add(self, input_name, cached_file):
self.infos[input_name] = dict(
cached_file = cached_file,
data_indices = cached_file.data_indices(),
data = None,
start_index = -1,
end_index = -1,
)
self.mixed_data_indices = mixDataIndices([ x['data_indices'] for x in self.infos.values() ])
self.nb_data_units = len(self.mixed_data_indices)
self.data_index = self.mixed_data_indices[0][0]
self.data_index_end = self.mixed_data_indices[-1][1]
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return 0
else:
return self.nb_data_units
def view(self, input_name, index):
if index < 0:
return None
try:
indices = self.infos[input_name]['data_indices'][index]
except:
return None
limited_data_indices = [ x for x in self.mixed_data_indices
if (indices[0] <= x[0]) and (x[1] <= indices[1]) ]
return DataView(self, limited_data_indices)
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.mixed_data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class InputList:
"""Represents the list of inputs of a processing block
......@@ -684,7 +543,7 @@ class InputList:
Attributes:
group (beat.core.inputs.InputGroup): Main group (for data-driven
main_group (beat.core.inputs.InputGroup): Main group (for data-driven
algorithms)
"""
......
File mode changed from 100755 to 100644
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, outputs):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, output):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, outputs):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, output):
return True
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import six
import nose.tools
from ..algorithm import Algorithm
from . import prefix
#----------------------------------------------------------
def test_load_unknown_algorithm():
algorithm = Algorithm(prefix, 'user/unknown/1')
assert algorithm.valid is False
assert algorithm.errors[0].find('file not found') != -1
#----------------------------------------------------------
@nose.tools.raises(SyntaxError)
def test_raises_syntax_error():
algorithm = Algorithm(prefix, 'user/syntax_error/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner() #this will effectively load the python code
#----------------------------------------------------------
@nose.tools.raises(RuntimeError)
def test_raises_runtime_error():
algorithm = Algorithm(prefix, 'user/syntax_error/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner(exc=RuntimeError)
#----------------------------------------------------------
@nose.tools.raises(AttributeError)
def test_missing_class_raises():
algorithm = Algorithm(prefix, 'user/no_class/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner()
#----------------------------------------------------------
def test_load_valid_algorithm():
algorithm = Algorithm(prefix, 'user/valid_algorithm/1')
nose.tools.eq_(algorithm.name, 'user/valid_algorithm/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
assert not algorithm.errors
assert not algorithm.results #it is not an analyzer
assert not algorithm.parameters #does not contain parameters
assert algorithm.input_map
assert algorithm.output_map
assert algorithm.schema_version is 1
runnable = algorithm.runner()