Open-source release

.gitignore

+*~
+*.swp
+*.pyc
+*.so
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
+*.egg-info
+develop-eggs
+sphinx
+doc/api
+dist
+.nfs*
+.gdb_history
+build
+*.egg
+opsnr.stt
+.coverage
+.DS_Store
+html/

AUTHORS.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.web module of the BEAT platform.             ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+===========================================================
+ Authors of the Biometrics Evaluation and Testing Platform
+===========================================================
+
+Andre Anjos <andre.anjos@idiap.ch>
+Flavio Tarsetti <flavio.tarsetti@idiap.ch>
+Laurent El-Shafey <laurent.el-shafey@idiap.ch>
+Philip Abbet <philip.abbet@idiap.ch>
+Samuel Gaist <samuel.gaist@idiap.ch>
+Sebastien Marcel <sebastien.marcel@idiap.ch>

MANIFEST.in

+include LICENSE.AGPL README.rst buildout.cfg bootstrap-buildout.py
+recursive-include doc conf.py *.rst *.png *.svg *.ico

README.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.backend.python module of the BEAT platform.  ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+
+============================================
+ Biometrics Evaluation and Testing Platform
+============================================
+
+This package contains the source code for a python-based backend for the BEAT
+platform.
+
+Installation
+------------
+
+Really easy, with ``zc.buildout``::
+
+  $ python bootstrap-buildout.py
+  $ ./bin/buildout
+
+These 2 commands should download and install all non-installed dependencies and
+get you a fully operational test and development environment.
+
+.. note::
+
+  If you are on the Idiap filesystem, you may use
+  ``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
+  package instead. It contains the same setup deployed at the final BEAT
+  machinery.
+
+
+Documentation
+-------------
+
+To build the documentation, just do::
+
+  $ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat/backend/python
+  $ ./bin/sphinx-build doc sphinx
+
+
+Testing
+-------
+
+After installation, it is possible to run our suite of unit tests. To do so,
+use ``nose``::
+
+  $ ./bin/nosetests -sv
+
+If you want to skip slow tests (at least those pulling stuff from our servers)
+or executing lengthy operations, just do::
+
+  $ ./bin/nosetests -sv -a '!slow'
+
+To measure the test coverage, do the following::
+
+  $ ./bin/nosetests -sv --with-coverage --cover-package=beat.backend.python
+
+To produce an HTML test coverage report, at the directory `./htmlcov`, do the
+following::
+
+  $ ./bin/nosetests -sv --with-coverage --cover-package=beat.backend.python --cover-html --cover-html-dir=htmlcov
+
+Our documentation is also interspersed with test units. You can run them using
+sphinx::
+
+  $ ./bin/sphinx -b doctest doc sphinx
+
+

beat/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+__import__('pkg_resources').declare_namespace(__name__)

beat/backend/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+__import__('pkg_resources').declare_namespace(__name__)

beat/backend/python/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+

beat/backend/python/executor.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+'''A class that can setup and execute algorithm blocks on the backend'''
+
+import logging
+logger = logging.getLogger(__name__)
+
+import os
+import time
+
+import zmq
+import simplejson
+
+from . import algorithm
+from . import inputs
+from . import outputs
+
+
+class Executor(object):
+  """Executors runs the code given an execution block information
+
+  Parameters:
+
+    socket (zmq.Socket): A pre-connected socket to send and receive messages
+      from.
+
+    directory (str): The path to a directory containing all the information
+      required to run the user experiment.
+
+    dataformat_cache (dict, optional): A dictionary mapping dataformat names to
+      loaded dataformats. This parameter is optional and, if passed, may
+      greatly speed-up database loading times as dataformats that are already
+      loaded may be re-used. If you use this parameter, you must guarantee that
+      the cache is refreshed as appropriate in case the underlying dataformats
+      change.
+
+    database_cache (dict, optional): A dictionary mapping database names to
+      loaded databases. This parameter is optional and, if passed, may
+      greatly speed-up database loading times as databases that are already
+      loaded may be re-used. If you use this parameter, you must guarantee that
+      the cache is refreshed as appropriate in case the underlying databases
+      change.
+
+    library_cache (dict, optional): A dictionary mapping library names to
+      loaded libraries. This parameter is optional and, if passed, may greatly
+      speed-up library loading times as libraries that are already loaded may
+      be re-used. If you use this parameter, you must guarantee that the cache
+      is refreshed as appropriate in case the underlying libraries change.
+  """
+
+  def __init__(self, socket, directory, dataformat_cache=None,
+          database_cache=None, library_cache=None):
+
+    self.socket = socket
+    self.comm_time = 0. #total communication time
+
+    self.configuration = os.path.join(directory, 'configuration.json')
+    with open(self.configuration, 'rb') as f: self.data = simplejson.load(f)
+    self.prefix = os.path.join(directory, 'prefix')
+    self.runner = None
+
+    # temporary caches, if the user has not set them, for performance
+    database_cache = database_cache if database_cache is not None else {}
+    dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
+    library_cache = library_cache if library_cache is not None else {}
+
+    self.algorithm = algorithm.Algorithm(self.prefix, self.data['algorithm'],
+            dataformat_cache, library_cache)
+
+    # Use algorithm names for inputs and outputs
+    main_channel = self.data['channel']
+
+    # Loads algorithm inputs
+    if 'inputs' in self.data:
+      self.input_list = inputs.InputList()
+      for name, channel in self.data['inputs'].items():
+        group = self.input_list.group(channel)
+        if group is None:
+          group = inputs.InputGroup(channel, (channel == main_channel),
+                      socket=self.socket)
+          self.input_list.add(group)
+        thisformat = self.algorithm.dataformats[self.algorithm.input_map[name]]
+        group.add(inputs.Input(name, thisformat, self.socket))
+      logger.debug("Loaded input list with %d group(s) and %d input(s)",
+          self.input_list.nbGroups(), len(self.input_list))
+
+    # Loads outputs
+    if 'outputs' in self.data:
+      self.output_list = outputs.OutputList()
+      for name, channel in self.data['outputs'].items():
+        thisformat = self.algorithm.dataformats[self.algorithm.output_map[name]]
+        self.output_list.add(outputs.Output(name, thisformat, self.socket))
+      logger.debug("Loaded output list with %d output(s)",
+          len(self.output_list))
+
+    # Loads results if it is an analyzer
+    if 'result' in self.data:
+      self.output_list = outputs.OutputList()
+      name = 'result'
+      # Retrieve dataformats in the JSON of the algorithm
+      analysis_format = self.algorithm.result_dataformat()
+      analysis_format.name = 'analysis:' + self.algorithm.name
+      self.output_list.add(outputs.Output(name, analysis_format, self.socket))
+      logger.debug("Loaded output list for analyzer (1 single output)")
+
+
+  def setup(self):
+    """Sets up the algorithm to start processing"""
+
+    self.runner = self.algorithm.runner()
+    retval = self.runner.setup(self.data['parameters'])
+    logger.debug("User algorithm is setup")
+    return retval
+
+
+  def process(self):
+    """Executes the user algorithm code using the current interpreter.
+    """
+
+    if not self.input_list or not self.output_list:
+      raise RuntimeError("I/O for execution block has not yet been set up")
+
+    using_output = self.output_list[0] if self.analysis else self.output_list
+
+    _start = time.time()
+
+    while self.input_list.hasMoreData():
+      main_group = self.input_list.main_group
+      main_group.restricted_access = False
+      main_group.next()
+      main_group.restricted_access = True
+      if not self.runner.process(self.input_list, using_output): return False
+
+    missing_data_outputs = [x for x in self.output_list if x.isDataMissing()]
+
+    proc_time = time.time() - _start
+
+    if missing_data_outputs:
+      raise RuntimeError("Missing data on the following output(s): %s" % \
+              ', '.join([x.name for x in missing_data_outputs]))
+
+    self.comm_time = sum([x.comm_time for x in self.input_list]) + \
+            sum([x.comm_time for x in self.output_list])
+    self.comm_time += sum([self.input_list[k].comm_time for k in range(self.input_list.nbGroups())])
+
+    # some local information
+    logger.debug("Total processing time was %.3f seconds" , proc_time)
+    logger.debug("Time spent in I/O was %.3f seconds" , self.comm_time)
+    logger.debug("I/O/Processing ratio is %d%%",
+            100*self.comm_time/proc_time)
+
+    # Handle the done command
+    self.done()
+
+    return True
+
+
+  def done(self):
+    """Indicates the infrastructure the execution is done"""
+
+    logger.debug('send: (don) done')
+    self.socket.send('don', zmq.SNDMORE)
+    self.socket.send('%.6e' % self.comm_time)
+    answer = self.socket.recv() #ack
+    logger.debug('recv: %s', answer)
+
+
+  @property
+  def schema_version(self):
+    """Returns the schema version"""
+    return self.data.get('schema_version', 1)
+
+
+  @property
+  def analysis(self):
+    """A boolean that indicates if the current block is an analysis block"""
+    return 'result' in self.data

beat/backend/python/library.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+"""Validation for libraries"""
+
+import os
+
+import simplejson
+
+from . import loader
+
+
+class Library(object):
+  """Librarys represent independent algorithm components within the platform.
+
+  This class can only parse the meta-parameters of the library. The actual
+  library is not directly treated by this class - only by the associated
+  algorithms.
+
+
+  Parameters:
+
+    prefix (str): Establishes the prefix of your installation.
+
+    name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
+
+    library_cache (dict, optional): A dictionary mapping library names to
+      loaded libraries. This parameter is optional and, if passed, may greatly
+      speed-up library loading times as libraries that are already loaded may
+      be re-used.
+
+
+  Attributes:
+
+    name (str): The library name
+
+    libraries (dict): A mapping object defining other libraries this library
+      needs to load so it can work properly.
+
+    uses (dict): A mapping object defining the required library import name
+      (keys) and the full-names (values).
+
+    data (dict): The original data for this library, as loaded by our JSON
+      decoder.
+
+    code (str): The code that is associated with this library, loaded as a
+      text (or binary) file.
+
+  """
+
+  def __init__(self, prefix, name, library_cache=None):
+
+    self.prefix = prefix
+
+    self.libraries = {}
+
+    library_cache = library_cache if library_cache is not None else {}
+
+    self.name = name
+    json_path = os.path.join(prefix, 'libraries', name + '.json')
+    with open(json_path, 'rb') as f: self.data = simplejson.load(f)
+
+    self.code_path = os.path.join(prefix, 'libraries', name + '.py')
+
+    # if no errors so far, make sense out of the library data
+    self.data.setdefault('uses', {})
+
+    if self.uses is not None:
+      for name, value in self.uses.items():
+        self.libraries[value] = Library(self.prefix, value, library_cache)
+
+    self.libraries[self.name] = self
+
+
+  def uses_dict(self):
+    """Returns the usage dictionary for all dependent modules"""
+
+    retval = {}
+
+    if self.uses is not None:
+
+      for name, value in self.uses.items():
+        retval[name] = dict(
+                path=self.libraries[value].code_path,
+                uses=self.libraries[value].uses_dict(),
+                )
+
+    return retval
+
+
+  def load(self):
+    """Loads the Python module for this library resolving all references
+
+    Returns the loaded Python module.
+    """
+
+    return loader.load_module(self.name.replace(os.sep, '_'),
+            self.code_path, self.uses_dict())
+
+
+  @property
+  def schema_version(self):
+    """Returns the schema version"""
+    return self.data.get('schema_version', 1)
+
+
+  @property
+  def uses(self):
+    return self.data.get('uses')

beat/backend/python/loader.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+'''Implements a simple loader for Python code'''
+
+import os
+import sys
+import six
+import imp
+
+
+def load_module(name, path, uses):
+  '''Loads the Python file as module, returns  a proper Python module
+
+
+  Parameters:
+
+    name (str): The name of the Python module to create. Must be a valid Python
+      symbol name
+
+    path (str): The full path of the Python file to load the module contents
+      from
+
+    uses (dict): A mapping which indicates the name of the library to load (as
+      a module for the current library) and the full-path and use mappings of
+      such modules.
+
+
+  Returns:
+
+    module: A valid Python module you can use in an Algorithm or Library.
+
+  '''
+
+  retval = imp.new_module(name)
+
+  #loads used modules
+  for k, v in uses.items():
+    retval.__dict__[k] = load_module(k, v['path'], v['uses'])
+
+  #execute the module code on the context of previously import modules
+  exec(compile(open(path, "rb").read(), path, 'exec'), retval.__dict__)
+
+  return retval
+
+
+def run(obj, method, exc=None, *args, **kwargs):
+  '''Runs a method on the object and protects its execution
+
+  In case an exception is raised, it is caught and transformed into the
+  exception class the user passed.
+
+  Parameters:
+
+    obj (object): The python object in which execute the method
+
+    method (str): The method name to execute on the object
+
+    exc (class, optional): The class to use as base exception when translating
+      the exception from the user code. If you set it to ``None``, then just
+      re-throws the user raised exception.
+
+    *args: Arguments to the object method, passed unchanged
+
+    **kwargs: Arguments to the object method, passed unchanged
+
+  Returns:
+
+    object: whatever ``obj.method()`` is bound to return.
+
+  '''
+
+  try:
+    if method == '__new__': return obj(*args, **kwargs)
+    return getattr(obj, method)(*args, **kwargs)
+  except Exception as e:
+    if exc is not None:
+      type, value, traceback = sys.exc_info()
+      six.reraise(exc, exc(value), traceback)
+    else:
+      raise #just re-raise the user exception

beat/backend/python/outputs.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.backend.python module of the BEAT platform.   #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+
+import six
+import zmq
+
+from .baseformat import baseformat
+
+
+class Output:
+  """Represents one output of a processing block
+
+  A list of outputs implementing this interface is provided to the algorithms
+  (see :py:class:`beat.backend.python.outputs.OutputList`).
+
+
+  Parameters:
+
+    name (str): Name of the output
+