Open-source release

.gitignore

+*~
+*.swp
+*.pyc
+*.so
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
+*.egg-info
+develop-eggs
+sphinx
+doc/api
+src
+dist
+.nfs*
+.gdb_history
+build
+*.egg
+opsnr.stt
+.coverage
+.DS_Store
+html/

.gitlab-ci.yml

+py27-linux:
+  script:
+  - git clean -ffdx
+  - export TMPDIR=/var/tmp
+  - /idiap/project/beat/environments/staging/usr/bin/python bootstrap-buildout.py
+  - ./bin/buildout
+  - ./bin/python --version
+  - unset TMPDIR
+  - cd src/cpulimit && make && cd -
+  - cd bin && ln -s ../src/cpulimit/src/cpulimit . && cd -
+  - export NOSE_WITH_COVERAGE=1
+  - export NOSE_COVER_PACKAGE=beat.core
+  - ./bin/nosetests -sv
+  - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
+  - ./bin/sphinx-build doc html
+  tags:
+  - lidiap2015
+
+py27-macosx:
+  script:
+  - git clean -ffdx
+  - /Users/buildbot/work/environments/beat/py27/bin/python bootstrap-buildout.py
+  - ./bin/buildout
+  - ./bin/python --version
+  - cd src/cpulimit && make && cd -
+  - cd bin && ln -s ../src/cpulimit/src/cpulimit . && cd -
+  - export NOSE_WITH_COVERAGE=1
+  - export NOSE_COVER_PACKAGE=beat.core
+  - ./bin/nosetests -sv
+  - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
+  - ./bin/sphinx-build doc html
+  tags:
+  - beat-macosx

AUTHORS.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.web module of the BEAT platform.             ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+===========================================================
+ Authors of the Biometrics Evaluation and Testing Platform
+===========================================================
+
+Andre Anjos <andre.anjos@idiap.ch>
+Flavio Tarsetti <flavio.tarsetti@idiap.ch>
+Laurent El-Shafey <laurent.el-shafey@idiap.ch>
+Philip Abbet <philip.abbet@idiap.ch>
+Samuel Gaist <samuel.gaist@idiap.ch>
+Sebastien Marcel <sebastien.marcel@idiap.ch>

MANIFEST.in

+include LICENSE.AGPL README.rst buildout.cfg bootstrap-buildout.py
+recursive-include doc conf.py *.rst *.png *.svg *.ico *.odg *.pdf *.dot
+recursive-include beat/core/schema *.json
+recursive-include beat/core/prototypes *.json *.py
+recursive-include beat/core/test/prefix *.json *.py

README.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.core module of the BEAT platform.            ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+
+============================================
+ Biometrics Evaluation and Testing Platform
+============================================
+
+This package contains the source code for the core components of the BEAT
+platform.
+
+Installation
+------------
+
+Really easy, with ``zc.buildout``::
+
+  $ python bootstrap-buildout.py
+  $ ./bin/buildout
+
+These 2 commands should download and install all non-installed dependencies and
+get you a fully operational test and development environment.
+
+.. note::
+
+  The python shell used in the first line of the previous command set
+  determines the python interpreter that will be used for all scripts developed
+  inside this package.
+
+  If you are on the Idiap filesystem, you may use
+  ``/idiap/project/beat/beat.env.deploy/usr/bin/python`` to bootstrap this
+  package instead. It contains the same setup deployed at the final BEAT
+  machinery.
+
+Cpulimit
+========
+
+Make sure the program ``cpulimit`` is available on your system or by the side
+of the python interpreter you bootstrapped as per instructions above. The BEAT
+platform uses this program to control slot usage on the scheduling/worker
+level::
+
+  $ cpulimit -h
+
+If that is not the case, then you need to install it. Either install a package
+that is native to your system (e.g. on Debian or Ubuntu platforms) or compile
+the checked-out version available at ``src/cpulimit``::
+
+  $ cd src/cpulimit;
+  $ make
+  $ ./src/cpulimit -h #to test it
+  $ cd ../../bin #go back to the root of beat.web and the into the `bin' dir
+  $ ln -s ../src/cpulimit/src/cpulimit
+  $ cd .. #go back to the root of beat.web
+
+
+Documentation
+-------------
+
+To build the documentation, just do::
+
+  $ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat beat/core/test beat/core/scripts
+  $ ./bin/sphinx-build doc sphinx
+
+
+Testing
+-------
+
+After installation, it is possible to run our suite of unit tests. To do so,
+use ``nose``::
+
+  $ ./bin/nosetests -sv
+
+
+.. note::
+
+   Some of the tests for our command-line toolkit require a running BEAT
+   platform web-server, with a compatible ``beat.core`` installed (preferably
+   the same).  By default, these tests will be skipped. If you want to run
+   them, you must setup a development web server and set the environment
+   variable ``BEAT_CORE_TEST_PLATFORM`` to point to that address. For example::
+
+     $ export BEAT_CORE_TEST_PLATFORM="http://example.com/platform/"
+     $ ./bin/nosetests -sv
+
+   It is **not** adviseable to run tests against a production web server.
+
+If you want to skip slow tests (at least those pulling stuff from our servers)
+or executing lengthy operations, just do::
+
+  $ ./bin/nosetests -sv -a '!slow'
+
+To measure the test coverage, do the following::
+
+  $ ./bin/nosetests -sv --with-coverage --cover-package=beat.core
+
+To produce an HTML test coverage report, at the directory `./htmlcov`, do the
+following::
+
+  $ ./bin/nosetests -sv --with-coverage --cover-package=beat.core --cover-html --cover-html-dir=htmlcov
+
+Our documentation is also interspersed with test units. You can run them using
+sphinx::
+
+  $ ./bin/sphinx -b doctest doc sphinx
+
+
+Development
+-----------
+
+
+Indentation
+===========
+
+You can enforce `PEP8 <https://www.python.org/dev/peps/pep-0008/>` compliance
+using the application ``autopep8``. For example, to enforce compliance on a
+single file and edit it in place, do::
+
+  $ ./bin/autopep8 --indent-size=2 --in-place beat/core/utils.py
+
+We normally use 2-space identattion. If ever, you can easily change the
+identation to 4 spaces like this::
+
+  $ ./bin/autopep8 --indent-size=4 --in-place beat/core/utils.py
+
+
+Profiling
+=========
+
+In order to profile the test code, try the following::
+
+  $ ./bin/python -mcProfile -oprof.data ./bin/nosetests -sv ...
+
+This will dump the profiling data at ``prof.data``. You can dump its contents
+in different ways using another command::
+
+  $ ./bin/python -mpstats prof.data
+
+This will allow you to dump and print the profiling statistics as you may find
+fit.

beat/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.core module of the BEAT platform.             #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+__import__('pkg_resources').declare_namespace(__name__)

beat/core/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.core module of the BEAT platform.             #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################

beat/core/agent.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.core module of the BEAT platform.             #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+import os
+import shutil
+import threading
+
+import logging
+logger = logging.getLogger(__name__)
+
+import zmq
+
+from . import utils
+from . import async
+from . import baseformat
+
+
+class Server(threading.Thread):
+  '''A 0MQ server for our communication with the user process'''
+
+  def __init__(self, input_list, output_list):
+
+    super(Server, self).__init__()
+    self._stop = threading.Event()
+
+    # Starts our 0MQ server
+    self.context = zmq.Context()
+    self.socket = self.context.socket(zmq.PAIR)
+    self.address = 'tcp://127.0.0.1'
+    port = self.socket.bind_to_random_port(self.address)
+    self.address += ':%d' % port
+    logger.debug("zmq server bound to `%s'", self.address)
+    self.poller = zmq.Poller()
+    self.poller.register(self.socket, zmq.POLLIN)
+
+    self.input_list = input_list
+    self.output_list = output_list
+
+    self.system_error = ''
+    self.user_error = ''
+    self.last_statistics = {}
+
+    # implementations
+    self.callbacks = dict(
+            nxt = self.next,
+            hmd = self.has_more_data,
+            idd = self.is_dataunit_done,
+            wrt = self.write,
+            idm = self.is_data_missing,
+            oic = self.output_is_connected,
+            don = self.done,
+            err = self.error,
+            )
+
+
+  def stop(self):
+    self._stop.set()
+
+
+  def stopped(self):
+    return self._stop.isSet()
+
+
+  def set_process(self, process):
+    self.process = process
+
+
+  def run(self):
+
+    logger.debug("0MQ server thread started")
+
+    while not self.stopped():
+
+      timeout = 1000 #ms
+      socks = dict(self.poller.poll(1000)) #blocks here, for 1 second at most
+
+      if self.stopped(): break #break immediately
+
+      if self.socket in socks and socks[self.socket] == zmq.POLLIN:
+
+        # incomming
+        more = True
+        parts = []
+        while more:
+          parts.append(self.socket.recv())
+          more = self.socket.getsockopt(zmq.RCVMORE)
+        command = parts[0]
+
+        logger.debug("recv: %s", command)
+
+        if command in self.callbacks:
+          try: #to handle command
+            self.callbacks[command](*parts[1:])
+          except:
+            import traceback
+            parser = lambda s: s if len(s)<20 else s[:20] + '...'
+            parsed_parts = ' '.join([parser(k) for k in parts])
+            message = "A problem occurred while performing command `%s' " \
+                    "killing user process. Exception:\n %s" % \
+                    (parsed_parts, traceback.format_exc())
+            logger.error(message, exc_info=True)
+            self.system_error = message
+            self.process.kill()
+            self.stop()
+            break
+
+        else:
+          message = "Command `%s' is not implemented - stopping user process" \
+                  % command
+          logger.error(message)
+          self.system_error = message
+          self.process.kill()
+          self.stop()
+          break
+
+    self.socket.setsockopt(zmq.LINGER, 0)
+    self.socket.close()
+    self.context.term()
+    logger.debug("0MQ server thread stopped")
+
+
+  def _get_input_candidate(self, channel, name):
+
+    channel_group = self.input_list.group(channel)
+    retval = channel_group[name]
+    if retval is None:
+        raise RuntimeError("Could not find input `%s' at channel `%s'" % \
+            (name, channel))
+    return retval
+
+
+  def _get_output_candidate(self, name):
+
+    retval = self.output_list[name]
+    if retval is None: raise RuntimeError("Could not find output `%s'" % name)
+    return retval
+
+
+  def next(self, channel, name=None):
+    """Syntax: nxt channel [name] ..."""
+
+    if name is not None: #single input
+      logger.debug('recv: nxt %s %s', channel, name)
+
+      input_candidate = self._get_input_candidate(channel, name)
+      input_candidate.next()
+      if input_candidate.data is None: #error
+          message = "User algorithm asked for more data for channel " \
+                  "`%s' on input `%s', but it is over (no more data). This " \
+                  "normally indicates a programming error on the user " \
+                  "side." % (channel, name)
+          self.user_error += message + '\n'
+          raise RuntimeError(message)
+      if isinstance(input_candidate.data, baseformat.baseformat):
+        packed = input_candidate.data.pack()
+      else:
+        packed = input_candidate.data
+      logger.debug('send: <bin> (size=%d)', len(packed))
+      self.socket.send(packed)
+
+    else: #whole group data
+      logger.debug('recv: nxt %s', channel)
+
+      channel_group = self.input_list.group(channel)
+
+      # Call next() on the group
+      channel_group.restricted_access = False
+      channel_group.next()
+      channel_group.restricted_access = True
+
+      # Loop over the inputs
+      inputs_to_go = len(channel_group)
+      self.socket.send(str(inputs_to_go), zmq.SNDMORE)
+      for inp in channel_group:
+        logger.debug('send: %s', inp.name)
+        self.socket.send(inp.name, zmq.SNDMORE)
+        if inp.data is None:
+          message = "User algorithm process asked for more data on channel " \
+                  "`%s' (all inputs), but input `%s' has nothing. This " \
+                  "normally indicates a programming error on the user " \
+                  "side." % (channel, inp.name)
+          self.user_error += message + '\n'
+          raise RuntimeError(message)
+        elif isinstance(inp.data, baseformat.baseformat):
+          packed = inp.data.pack()
+        else:
+          packed = inp.data
+        logger.debug('send: <bin> (size=%d)', len(packed))
+        inputs_to_go -= 1
+        if inputs_to_go > 0:
+          self.socket.send(packed, zmq.SNDMORE)
+        else:
+          self.socket.send(packed)
+
+
+  def has_more_data(self, channel, name=None):
+    """Syntax: hmd channel [name]"""
+
+    if name: #single input
+      logger.debug('recv: hmd %s %s', channel, name)
+      input_candidate = self._get_input_candidate(channel, name)
+      what = 'tru' if input_candidate.hasMoreData() else 'fal'
+
+    else: #for all channel names
+      logger.debug('recv: hmd %s', channel)
+      channel_group = self.input_list.group(channel)
+      what = 'tru' if channel_group.hasMoreData() else 'fal'
+
+    logger.debug('send: %s', what)
+    self.socket.send(what)
+
+
+  def is_dataunit_done(self, channel, name):
+    """Syntax: idd channel name"""
+
+    logger.debug('recv: idd %s %s', channel, name)
+    input_candidate = self._get_input_candidate(channel, name)
+    what = 'tru' if input_candidate.isDataUnitDone() else 'fal'
+    logger.debug('send: %s', what)
+    self.socket.send(what)
+
+
+  def write(self, name, packed):
+    """Syntax: wrt output data"""
+
+    logger.debug('recv: wrt %s <bin> (size=%d)', name, len(packed))
+
+    # Get output object
+    output_candidate = self._get_output_candidate(name)
+    if output_candidate is None:
+      raise RuntimeError("Could not find output `%s' to write to" % name)
+
+    data = output_candidate.data_sink.dataformat.type()
+    data.unpack(packed)
+    output_candidate.write(data)
+
+    logger.debug('send: ack')
+    self.socket.send('ack')
+
+
+  def is_data_missing(self, name):
+    """Syntax: idm output"""
+
+    logger.debug('recv: idm %s', name)
+
+    output_candidate = self._get_output_candidate(name)
+    what = 'tru' if output_candidate.isDataMissing() else 'fal'
+    logger.debug('send: %s', what)
+    self.socket.send(what)
+
+
+  def output_is_connected(self, name):
+    """Syntax: oic output"""
+
+    logger.debug('recv: oic %s', name)
+
+    output_candidate = self._get_output_candidate(name)
+    what = 'tru' if output_candidate.isConnected() else 'fal'
+    logger.debug('send: %s', what)
+    self.socket.send(what)
+
+
+  def _collect_statistics(self):
+
+    logger.debug('collecting user process statistics...')
+    self.last_statistics = self.process.statistics()
+
+
+  def _acknowledge(self):
+
+    logger.debug('send: ack')
+    self.socket.send('ack')
+    logger.debug('setting stop condition for 0MQ server thread')
+    self.stop()
+
+
+  def done(self, wait_time):
+    """Syntax: don"""
+