Skip to content

beat.backend.python
beat.backend.python
Commit 3fff4381 authored by Philip ABBET's avatar Philip ABBET
Browse files
Options

New API for the algorithms

parent ec74eae8
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1698 additions and 1133 deletions
beat/backend/python/algorithm.py 100644 → 100755
View file @ 3fff4381
......@@ -41,6 +41,8 @@ from . import loader
from . import utils
#----------------------------------------------------------
class Storage(utils.CodeStorage):
"""Resolves paths for algorithms
......@@ -67,6 +69,8 @@ class Storage(utils.CodeStorage):
super(Storage, self).__init__(path, language)
#----------------------------------------------------------
class Runner(object):
'''A special loader class for algorithms, with specialized methods
......@@ -79,13 +83,13 @@ class Runner(object):
obj_name (str): The name of the object within the module you're interested
on
algorithm (object): The algorithm instance that is used for parameter
checking.
exc (class): The class to use as base exception when translating the
exception from the user code. Read the documention of :py:func:`run`
for more details.
algorithm (object): The algorithm instance that is used for parameter
checking.
*args: Constructor parameters for the algorithm (normally none)
**kwargs: Constructor parameters for the algorithm (normally none)
......@@ -125,17 +129,15 @@ class Runner(object):
if not user_keys.issubset(valid_keys):
err_keys = user_keys - valid_keys
message = "parameters `%s' are not declared for algorithm `%s' - " \
"valid parameters are `%s'" % (
"valid parameters are `%s'" % (
','.join(err_keys),
self.name,
','.join(valid_keys),
)
)
exc = self.exc or KeyError
raise exc(message)
# checks all values set by the user are in range (if a range is set)
retval = dict() #dictionary with checked user parameters and defaults
retval = dict()
for key, definition in algo_parameters.items():
......@@ -160,17 +162,29 @@ class Runner(object):
def setup(self, parameters, *args, **kwargs):
'''Sets up the algorithm, only effective on the first call'''
if self.ready: return self.ready
if self.ready:
return self.ready
completed_parameters = self._check_parameters(parameters) #may raise
completed_parameters = self._check_parameters(parameters)
kwargs['parameters'] = completed_parameters
if hasattr(self.obj, 'setup'):
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
return self.ready
else:
self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
return self.ready
def prepare(self, parameters, *args, **kwargs):
'''Let the algorithm process the data on the non-principal channels'''
if not self.ready:
message = "algorithm `%s' is not yet setup" % (self.name,)
exc = self.exc or RuntimeError
raise self.exc(message)
if not hasattr(self.obj, 'prepare'):
return True
return loader.run(self.obj, 'prepare', self.exc, *args, **kwargs)
def process(self, *args, **kwargs):
'''Runs through data'''
......@@ -188,6 +202,8 @@ class Runner(object):
return getattr(self.obj, key)
#----------------------------------------------------------
class Algorithm(object):
"""Algorithms represent runnable components within the platform.
......@@ -262,6 +278,11 @@ class Algorithm(object):
"""
LEGACY = 'legacy'
SEQUENTIAL = 'sequential'
AUTONOMOUS = 'autonomous'
def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
self._name = None
......@@ -451,6 +472,21 @@ class Algorithm(object):
return self.data['language']
@property
def api_version(self):
"""Returns the API version"""
return self.data.get('api_version', 1)
@property
def type(self):
"""Returns the type of algorithm"""
if self.api_version == 1:
return Algorithm.LEGACY
return self.data.get('type', Algorithm.SEQUENTIAL)
@language.setter
def language(self, value):
"""Sets the current executable code programming language"""
......
beat/backend/python/data_loaders.py 0 → 100755
View file @ 3fff4381
This diff is collapsed.
beat/backend/python/inputs.py
View file @ 3fff4381
......@@ -495,147 +495,6 @@ class InputGroup:
#----------------------------------------------------------
class DataView(object):
def __init__(self, data_loader_group, data_indices):
self.infos = {}
self.data_indices = data_indices
self.nb_data_units = len(data_indices)
self.data_index = data_indices[0][0]
self.data_index_end = data_indices[-1][1]
for input_name, infos in data_loader_group.infos.items():
input_data_indices = []
current_start = self.data_index
for i in range(self.data_index, self.data_index_end + 1):
for indices in infos['data_indices']:
if indices[1] == i:
input_data_indices.append( (current_start, i) )
current_start = i + 1
break
if (len(input_data_indices) == 0) or (input_data_indices[-1][1] != self.data_index_end):
input_data_indices.append( (current_start, self.data_index_end) )
self.infos[input_name] = dict(
cached_file = infos['cached_file'],
data_indices = input_data_indices,
data = None,
start_index = -1,
end_index = -1,
)
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return None
else:
return self.nb_data_units
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class DataLoaderGroup(object):
def __init__(self, channel):
self.channel = str(channel)
self.infos = {}
self.mixed_data_indices = None
self.nb_data_units = 0
self.data_index = -1 # Lower index across all inputs
self.data_index_end = -1 # Bigger index across all inputs
def add(self, input_name, cached_file):
self.infos[input_name] = dict(
cached_file = cached_file,
data_indices = cached_file.data_indices(),
data = None,
start_index = -1,
end_index = -1,
)
self.mixed_data_indices = mixDataIndices([ x['data_indices'] for x in self.infos.values() ])
self.nb_data_units = len(self.mixed_data_indices)
self.data_index = self.mixed_data_indices[0][0]
self.data_index_end = self.mixed_data_indices[-1][1]
def count(self, input_name=None):
if input_name is not None:
try:
return len(self.infos[input_name]['data_indices'])
except:
return 0
else:
return self.nb_data_units
def view(self, input_name, index):
if index < 0:
return None
try:
indices = self.infos[input_name]['data_indices'][index]
except:
return None
limited_data_indices = [ x for x in self.mixed_data_indices
if (indices[0] <= x[0]) and (x[1] <= indices[1]) ]
return DataView(self, limited_data_indices)
def __getitem__(self, index):
if index < 0:
return (None, None, None)
try:
indices = self.mixed_data_indices[index]
except:
return (None, None, None)
result = {}
for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \
infos['cached_file'].getAtDataIndex(indices[0])
result[input_name] = infos['data']
return (result, indices[0], indices[1])
#----------------------------------------------------------
class InputList:
"""Represents the list of inputs of a processing block
......@@ -684,7 +543,7 @@ class InputList:
Attributes:
group (beat.core.inputs.InputGroup): Main group (for data-driven
main_group (beat.core.inputs.InputGroup): Main group (for data-driven
algorithms)
"""
......
beat/backend/python/test/mocks.py 100755 → 100644
View file @ 3fff4381
File mode changed from 100755 to 100644
beat/backend/python/test/prefix/algorithms/autonomous/valid_algorithm/1.json 0 → 100644
View file @ 3fff4381
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
beat/backend/python/test/prefix/algorithms/autonomous/valid_algorithm/1.py 0 → 100755
View file @ 3fff4381
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, outputs):
return True
beat/backend/python/test/prefix/algorithms/autonomous/valid_analysis/1.json 0 → 100644
View file @ 3fff4381
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "autonomous",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
beat/backend/python/test/prefix/algorithms/autonomous/valid_analysis/1.py 0 → 100755
View file @ 3fff4381
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, data_loaders, output):
return True
beat/backend/python/test/prefix/algorithms/sequential/valid_algorithm/1.json 0 → 100644
View file @ 3fff4381
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"splittable": false,
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
},
"outputs": {
"features": {
"type": "user/1d_array_of_integers/1"
}
}
}
]
}
beat/backend/python/test/prefix/algorithms/sequential/valid_algorithm/1.py 0 → 100755
View file @ 3fff4381
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, outputs):
return True
beat/backend/python/test/prefix/algorithms/sequential/valid_analysis/1.json 0 → 100644
View file @ 3fff4381
{
"schema_version": 2,
"language": "python",
"api_version": 2,
"type": "sequential",
"groups": [
{
"inputs": {
"images": {
"type": "user/2d_array_of_integers/1"
},
"labels": {
"type": "user/floats/1"
}
}
}
],
"results": {
"c": {
"type": "plot/scatter/1",
"display": false
},
"v": {
"type": "float32",
"display": true
}
}
}
beat/backend/python/test/prefix/algorithms/sequential/valid_analysis/1.py 0 → 100755
View file @ 3fff4381
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
class Algorithm:
def setup(self, parameters):
return True
def prepare(self, data_loaders):
return True
def process(self, inputs, data_loaders, output):
return True
beat/backend/python/test/test_algorithm.py 0 → 100644
View file @ 3fff4381
This diff is collapsed.
beat/backend/python/test/test_algorithm_loading.py deleted 100644 → 0
View file @ ec74eae8
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import six
import nose.tools
from ..algorithm import Algorithm
from . import prefix
#----------------------------------------------------------
def test_load_unknown_algorithm():
algorithm = Algorithm(prefix, 'user/unknown/1')
assert algorithm.valid is False
assert algorithm.errors[0].find('file not found') != -1
#----------------------------------------------------------
@nose.tools.raises(SyntaxError)
def test_raises_syntax_error():
algorithm = Algorithm(prefix, 'user/syntax_error/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner() #this will effectively load the python code
#----------------------------------------------------------
@nose.tools.raises(RuntimeError)
def test_raises_runtime_error():
algorithm = Algorithm(prefix, 'user/syntax_error/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner(exc=RuntimeError)
#----------------------------------------------------------
@nose.tools.raises(AttributeError)
def test_missing_class_raises():
algorithm = Algorithm(prefix, 'user/no_class/1')
assert algorithm.valid, '\n * %s' % '\n * '.join(algorithm.errors)
runnable = algorithm.runner()
#----------------------------------------------------------
def test_load_valid_algorithm():