Commit 3fff4381 authored by Philip ABBET's avatar Philip ABBET

New API for the algorithms

parent ec74eae8
...@@ -41,6 +41,8 @@ from . import loader ...@@ -41,6 +41,8 @@ from . import loader
from . import utils from . import utils
#----------------------------------------------------------
class Storage(utils.CodeStorage): class Storage(utils.CodeStorage):
"""Resolves paths for algorithms """Resolves paths for algorithms
...@@ -67,6 +69,8 @@ class Storage(utils.CodeStorage): ...@@ -67,6 +69,8 @@ class Storage(utils.CodeStorage):
super(Storage, self).__init__(path, language) super(Storage, self).__init__(path, language)
#----------------------------------------------------------
class Runner(object): class Runner(object):
'''A special loader class for algorithms, with specialized methods '''A special loader class for algorithms, with specialized methods
...@@ -79,13 +83,13 @@ class Runner(object): ...@@ -79,13 +83,13 @@ class Runner(object):
obj_name (str): The name of the object within the module you're interested obj_name (str): The name of the object within the module you're interested
on on
algorithm (object): The algorithm instance that is used for parameter
checking.
exc (class): The class to use as base exception when translating the exc (class): The class to use as base exception when translating the
exception from the user code. Read the documention of :py:func:`run` exception from the user code. Read the documention of :py:func:`run`
for more details. for more details.
algorithm (object): The algorithm instance that is used for parameter
checking.
*args: Constructor parameters for the algorithm (normally none) *args: Constructor parameters for the algorithm (normally none)
**kwargs: Constructor parameters for the algorithm (normally none) **kwargs: Constructor parameters for the algorithm (normally none)
...@@ -125,17 +129,15 @@ class Runner(object): ...@@ -125,17 +129,15 @@ class Runner(object):
if not user_keys.issubset(valid_keys): if not user_keys.issubset(valid_keys):
err_keys = user_keys - valid_keys err_keys = user_keys - valid_keys
message = "parameters `%s' are not declared for algorithm `%s' - " \ message = "parameters `%s' are not declared for algorithm `%s' - " \
"valid parameters are `%s'" % ( "valid parameters are `%s'" % (
','.join(err_keys), ','.join(err_keys),
self.name, self.name,
','.join(valid_keys), ','.join(valid_keys),
) )
exc = self.exc or KeyError exc = self.exc or KeyError
raise exc(message) raise exc(message)
# checks all values set by the user are in range (if a range is set) retval = dict()
retval = dict() #dictionary with checked user parameters and defaults
for key, definition in algo_parameters.items(): for key, definition in algo_parameters.items():
...@@ -160,17 +162,29 @@ class Runner(object): ...@@ -160,17 +162,29 @@ class Runner(object):
def setup(self, parameters, *args, **kwargs): def setup(self, parameters, *args, **kwargs):
'''Sets up the algorithm, only effective on the first call''' '''Sets up the algorithm, only effective on the first call'''
if self.ready: return self.ready if self.ready:
return self.ready
completed_parameters = self._check_parameters(parameters) #may raise completed_parameters = self._check_parameters(parameters)
kwargs['parameters'] = completed_parameters kwargs['parameters'] = completed_parameters
if hasattr(self.obj, 'setup'): self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs) return self.ready
return self.ready
else:
def prepare(self, parameters, *args, **kwargs):
'''Let the algorithm process the data on the non-principal channels'''
if not self.ready:
message = "algorithm `%s' is not yet setup" % (self.name,)
exc = self.exc or RuntimeError
raise self.exc(message)
if not hasattr(self.obj, 'prepare'):
return True return True
return loader.run(self.obj, 'prepare', self.exc, *args, **kwargs)
def process(self, *args, **kwargs): def process(self, *args, **kwargs):
'''Runs through data''' '''Runs through data'''
...@@ -188,6 +202,8 @@ class Runner(object): ...@@ -188,6 +202,8 @@ class Runner(object):
return getattr(self.obj, key) return getattr(self.obj, key)
#----------------------------------------------------------
class Algorithm(object): class Algorithm(object):
"""Algorithms represent runnable components within the platform. """Algorithms represent runnable components within the platform.
...@@ -262,6 +278,11 @@ class Algorithm(object): ...@@ -262,6 +278,11 @@ class Algorithm(object):
""" """
LEGACY = 'legacy'
SEQUENTIAL = 'sequential'
AUTONOMOUS = 'autonomous'
def __init__(self, prefix, name, dataformat_cache=None, library_cache=None): def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
self._name = None self._name = None
...@@ -451,6 +472,21 @@ class Algorithm(object): ...@@ -451,6 +472,21 @@ class Algorithm(object):
return self.data['language'] return self.data['language']
@property
def api_version(self):
"""Returns the API version"""
return self.data.get('api_version', 1)
@property
def type(self):
"""Returns the type of algorithm"""
if self.api_version == 1:
return Algorithm.LEGACY
return self.data.get('type', Algorithm.SEQUENTIAL)
@language.setter @language.setter
def language(self, value): def language(self, value):
"""Sets the current executable code programming language""" """Sets the current executable code programming language"""
......
This diff is collapsed.
...@@ -495,147 +495,6 @@ class InputGroup: ...@@ -495,147 +495,6 @@ class InputGroup:
#---------------------------------------------------------- #----------------------------------------------------------
class DataView(object):
def __init__(self, data_loader_group, data_indices):
self.infos = {}
self.data_indices = data_indices
self.nb_data_units = len(data_indices)
self.data_index = data_indices[0][0]
self.data_index_end = data_indices[-1][1]
for input_name, infos in data_loader_group.infos.items():
input_data_indices = []
current_start = self.data_index
for i in range(self.data_index, self.data_index_end + 1):
for indices in infos['data_indices']:
if indices[1] == i:
input_data_indices.append( (current_start, i) )
current_start = i + 1
break
if (len(input_data_indices) == 0) or (input_data_indices[-1][1] != self.data_index_end):
input_data_indices.append( (current_start, self.data_index_end) )
self.infos[input_name] = dict(
cached_file = infos['cached_file'],
data_indices = input_data_indices,
data = None,
start_index = -1,
end_index = -1,
)
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return None
else:
return self.nb_data_units
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class DataLoaderGroup(object):
def __init__(self, channel):
self.channel = str(channel)
self.infos = {}
self.mixed_data_indices = None
self.nb_data_units = 0
self.data_index = -1 # Lower index across all inputs
self.data_index_end = -1 # Bigger index across all inputs
def add(self, input_name, cached_file):
self.infos[input_name] = dict(
cached_file = cached_file,
data_indices = cached_file.data_indices(),
data = None,
start_index = -1,
end_index = -1,
)
self.mixed_data_indices = mixDataIndices([ x['data_indices'] for x in self.infos.values() ])
self.nb_data_units = len(self.mixed_data_indices)
self.data_index = self.mixed_data_indices[0][0]
self.data_index_end = self.mixed_data_indices[-1][1]
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return 0
else:
return self.nb_data_units
def view(self, input_name, index):
if index < 0:
return None
try:
indices = self.infos[input_name]['data_indices'][index]
except:
return None
limited_data_indices = [ x for x in self.mixed_data_indices
if (indices[0] <= x[0]) and (x[1] <= indices[1]) ]
return DataView(self, limited_data_indices)
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.mixed_data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class InputList: class InputList:
"""Represents the list of inputs of a processing block """Represents the list of inputs of a processing block
...@@ -684,7 +543,7 @@ class InputList: ...@@ -684,7 +543,7 @@ class InputList:
Attributes: Attributes:
group (beat.core.inputs.InputGroup): Main group (for data-driven main_group (beat.core.inputs.InputGroup): Main group (for data-driven
algorithms) algorithms)
""" """
......
File mode changed from 100755 to 100644
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, outputs):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, output):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, outputs):
return True
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, output):
return True
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import six
import nose.tools
from ..algorithm import Algorithm
from .