Open-source release

.gitignore

+*~
+*.pyc
+*.swp
+*.swl
+*.swm
+*.swn
+*.swo
+parts/
+*.egg-info/
+develop-eggs/
+eggs/
+bin/
+sphinx/
+.installed.cfg
+.mr.developer.cfg
+.coverage
+*.sql3
+.DS_Store
+beat/web/settings/settings.py
+src/
+web_dynamic_data*
+.nfs*
+beat/web/static/
+doc/api/api/
+html/
+*.tar.bz2

.gitlab-ci.yml

+py27-linux:
+  script:
+  - git clean -ffdx
+  - export TMPDIR=/var/tmp
+  - /idiap/project/beat/environments/staging/usr/bin/python bootstrap-buildout.py
+  - ./bin/buildout
+  - ./bin/python --version
+  - unset TMPDIR
+  - export NOSE_WITH_COVERAGE=1
+  - export NOSE_COVER_PACKAGE=beat.web
+  - export COVERAGE_FILE=.coverage.django
+  - ./bin/django test --settings=beat.web.settings.test -v 2
+  - export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
+  - export COVERAGE_FILE=.coverage.cmdline
+  - ./bin/nosetests -sv beat.cmdline
+  - unset COVERAGE_FILE
+  - ./bin/coverage combine
+  - ./bin/coverage report
+  - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
+  - ./bin/sphinx-build doc/api html/api
+  - ./bin/sphinx-build doc/admin html/admin
+  - ./bin/sphinx-build doc/user html/user
+
+  tags:
+  - lidiap2015
+
+py27-macosx:
+  script:
+  - git clean -ffdx
+  - /Users/buildbot/work/environments/beat/py27/bin/python bootstrap-buildout.py
+  - ./bin/buildout
+  - ./bin/python --version
+  - export NOSE_WITH_COVERAGE=1
+  - export NOSE_COVER_PACKAGE=beat.web
+  - export COVERAGE_FILE=.coverage.django
+  - ./bin/django test --settings=beat.web.settings.test -v 2
+  - export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
+  - export COVERAGE_FILE=.coverage.cmdline
+  - ./bin/nosetests -sv beat.cmdline
+  - unset COVERAGE_FILE
+  - ./bin/coverage combine
+  - ./bin/coverage report
+  - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
+  - ./bin/sphinx-build doc/api html/api
+  - ./bin/sphinx-build doc/admin html/admin
+  - ./bin/sphinx-build doc/user html/user
+  tags:
+  - beat-macosx

AUTHORS.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.web module of the BEAT platform.             ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+===========================================================
+ Authors of the Biometrics Evaluation and Testing Platform
+===========================================================
+
+Andre Anjos <andre.anjos@idiap.ch>
+Flavio Tarsetti <flavio.tarsetti@idiap.ch>
+Laurent El-Shafey <laurent.el-shafey@idiap.ch>
+Philip Abbet <philip.abbet@idiap.ch>
+Samuel Gaist <samuel.gaist@idiap.ch>
+Sebastien Marcel <sebastien.marcel@idiap.ch>

MANIFEST.in

+include LICENSE.AGPL README.rst bootstrap-buildout.py buildout.cfg
+recursive-include beat/web *.css *.png *.gif *.svg *.ico *.js *.html *.rst *.txt
+recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf

README.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.web module of the BEAT platform.             ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+
+============================================
+ Biometrics Evaluation and Testing Platform
+============================================
+
+This package contains the source code for the web components of the BEAT
+platform.
+
+
+Installation
+------------
+
+Here is a recipe to get you started::
+
+  $ python bootstrap-buildout.py
+  $ ./bin/buildout
+
+These 2 commands should download and install all non-installed dependencies and
+get you a fully operational test and development environment.
+
+.. note::
+
+  The python shell used in the first line of the previous command set
+  determines the python interpreter that will be used for all scripts developed
+  inside this package.
+
+  If you are on the Idiap filesystem, you may use
+  ``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
+  package instead. It contains the same setup deployed at the final BEAT
+  machinery.
+
+
+.. tip::
+
+  If you'd like to **speed-up** the installation, it is strongly advised you
+  prepare a preset virtual environment (see the virtualenv_ package) with all
+  required dependencies, so that ``./bin/buildout`` does not download and
+  installs all of them every time you cleanup. This technique should allow you
+  to quickly clean-up and re-start your working environment which is useful
+  during development.
+
+  In order to fetch currently needed dependencies, run::
+
+    $ ./bin/buildout #to setup once
+    $ ./bin/pip freeze > requirements.txt
+
+  Examine the file ``requirements.txt`` and remove packages you are either
+  developing locally (e.g., all that are under ``src``) or that you think you
+  don't need. The command ``pip freeze`` reports all installed packages and not
+  only those which are needed by your project. If the Python prompt you used
+  for bootstrapping already had a good set of packages installed, you may see
+  them there.
+
+  Once you have a satisfying ``requirements.txt`` file, you may proceed to
+  recreate a virtualenv_ you'll use for your development. Just call::
+
+    $ virtualenv ~/work/beat-env #--system-site-packages
+
+  To create the virtual environment. This new environment does not contain
+  system packages by default. You may override that by specifying
+  ``--system-site-packages`` as suggested above. Then, install the required
+  packages on your new virtual environment::
+
+    $ ~/work/beat-env/bin/pip install -r requirements.txt
+
+  After that step is done, your virtual environment is ready for deployment.
+  You may now start from scratch to develop ``beat.web`` taking as base the
+  Python interpreter on your virtualenv_::
+
+    $ cd beat.web
+    $ git clean -fdx #full clean-up
+    $ ~/work/beat-env/bin/python bootstrap-buildout.py
+    $ ./bin/buildout
+
+  You'll realize the buildout step now takes considerably less time and you may
+  repeat this last step as much as needed. ``pip`` is a very flexible tool and
+  you may use it to manage the virtualenv_ installing and removing packages as
+  needed.
+
+
+Documentation
+-------------
+
+Our documentation project is divided in 3 parts. The user guide is the only one
+which is automatically built as part of the ``buildout`` procedure. The API and
+administrators guide needs to be manually compiled if required.
+
+To build the API documentation, just do::
+
+  $ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
+  $ ./bin/sphinx-build doc/api html/api
+
+
+To build the administrator guide, just do::
+
+  $ ./bin/sphinx-build doc/admin html/admin
+
+
+The above commands will build the stated guides, in HTML format, and dump
+results into your local directory ``html``. You may navigate then to that
+directory and, with your preferred web browser, open the file ``index.html`` to
+browse the available documentation.
+
+The basic user guide which includes information for users of the platform, is
+built automatically upon ``buildout``. If you wish to build it and place it
+alongside the other guides, you may do it as well like this::
+
+  $ ./bin/sphinx-build doc/user html/user
+
+
+Instantiating a BEAT web server
+-------------------------------
+
+For a simple (development) web server, the default settings on
+``beat/web/settings/settings.py`` should work out of the box. These settings:
+
+  * Instantiate the web service on the local host under port 8000 (the address
+    will be ``http://127.0.0.1:8000``
+  * Use an SQLITE3 database named ``django.sql3`` located on the current
+    working directory
+  * Run with full debug output
+  * It sets the working BEAT prefix to ``./prefix``
+  * A single user, called ``admin`` will be setup into the system
+
+If you need to tweak these settings, just edit the file
+``beat/web/settings/settings.py``. You may consult the `Django documentation`_
+for detailed information on other settings.
+
+Once the Django settings are in place, you can run a single command to fully
+populate a development webserver::
+
+  $ ./bin/django install -v1
+
+.. note::
+
+   Concerning databases installed by this command, we only explain the platform
+   how to **access** their data. It does not download the raw data for the
+   databases that you must procure yourself through the relevant web sites
+   (checkout the database pages on the Idiap instance of the BEAT platform for
+   details).
+
+.. note::
+
+  If you need to specify your own path to the directories containing the
+  databases, you could just create a simple JSON file as follows::
+
+    {
+      "atnt/1": "/remote/databases/atnt",
+      "banca/2": "/remote/databases/banca"
+    }
+
+  Then just use the previous script with the option ``--database-root-file``::
+
+    $ ./bin/django install -v1 --database-root-file=MYFILE.json
+
+  By default, paths to the root of all databases are set to match the Idiap
+  Research Institute filesystem organisation.
+
+.. note::
+
+  For every installed database, you'll need to generate their data indices,
+  which allows the platform to correctly parallelize algorithms. To do so, for
+  every combination of database and version you wish to support, run the
+  following command::
+
+    $ ./bin/beat -p web_dynamic_data db index <name>/<version>
+
+  Replacing the strings ``<name>`` by the name of the database you wish to dump
+  the indices for, together with the version in ``<version>``. For example, to
+  dump the indices for the AT&T database, version 1, do the following::
+
+    $ ./bin/beat -p web_dynamic_data db index atnt/1
+
+Once the contributions and users are in place, you're ready to start the test
+server::
+
+  $ ./bin/django runserver -v3
+
+At this point, your platform can be accessed by typing the URL
+``http://127.0.0.1:8000`` in a web browser on the machine the server is
+running.
+
+.. _localhost:
+
+Localhost
+---------
+
+To effectively use your new server and test all aspects of it, you'll also need
+a scheduler with at least one attached worker that can execute experiments. For
+most development purposes, a simple 3-node system, with all components running
+on the current (local) host is sufficient.
+
+Here is a recipe to start a simple 3-node system in which the local worker uses
+the system-wide installed Python interpreter to execute the algorithms.
+
+First, make sure the program ``cpulimit`` is available on your system. The BEAT
+platform uses this program to control slot usage on the scheduling/worker
+level::
+
+  $ cpulimit -h
+
+If that is not the case, then you need to install it. Either install a package
+that is native to your system (e.g. on Debian or Ubuntu platforms) or compile
+the checked-out version available at ``src/cpulimit``::
+
+  $ cd src/cpulimit;
+  $ make
+  $ ./src/cpulimit -h #to test it
+  $ cd ../../bin #go back to the root of beat.web and the into the `bin' dir
+  $ ln -s ../src/cpulimit/src/cpulimit
+  $ cd .. #go back to the root of beat.web
+
+Now start the localhost system::
+
+  $ ./bin/localhost.py -v
+  ...
+
+You may inspect this programs help message for details on its usage and
+options.
+
+Once the localhost system is started and the scheduler is properly configured,
+you may open a browser window to your `localhost, port 8000
+<http://127.0.0.1:8000>`_, to get started with your locally installed platform.
+
+
+Localhost with ``DEBUG=False``
+==============================
+
+If you need to test the RESTful API, it is better to do it without Django
+throwing you HTML error pages. For that, you'll need to start the Django
+development server with slightly different settings::
+
+  $ ./bin/localhost.py -v --settings=beat.web.settings.nodebug
+
+
+Triggering a Scheduler Reconfiguration
+======================================
+
+If you modify the queue configuration on the Django administrative panel,
+you'll need to notify the scheduler of those changes. You can trigger a
+scheduler (hot) re-configuration using the following command-line program::
+
+  $ ./bin/django qconf
+
+.. note::
+
+   Optionally, you may also visit `your local scheduler page
+   <http://127.0.0.1:8000/backend/scheduler>`, and hit the (green) button that
+   says "Send configuration to Scheduler". It has the same effect.
+
+
+Unit Testing
+------------
+
+After installation, it is possible to run our suite of unit tests. To do so,
+use::
+
+  $ ./bin/django test --settings=beat.web.settings.test -v 2
+
+You may pass filtering criteria to just launch tests for a particular set of
+``beat.web`` applications. For example, to run tests only concerning
+``beat.web.toolchains``, run::
+
+  $ ./bin/django test --settings=beat.web.settings.test -v 2 beat.web.toolchains.tests
+
+To measure coverage, you must set an environment variable for nose::
+
+  $ NOSE_WITH_COVERAGE=1 NOSE_COVER_PACKAGE=beat.web ./bin/django test --settings=beat.web.settings.test -v 2
+
+.. _snapshot:
+
+Local Development Server
+------------------------
+
+It is easy to quickly setup a local system for development, taking as base the
+current state of a production system.
+
+
+1. Before starting, make sure you have gone through, at least once, over the
+   localhost_ instructions above. It explains the very basic setup required for
+   a complete development environment.
+
+
+2. Dump and back-up your current **production** BEAT database::
+
+     [production]$ ./bin/django backup
+
+
+3. [Optional] If you have made important modifications between the contents
+   available at your production server and your currently checked-out source,
+   you'll need to run Django migrations on data imported from the production
+   server. If you need to do this, make sure you don't have unapplied commits
+   to your local **development** package and reset it to the production tag::
+
+     [development]$ git checkout <production-tag>
+
+   .. note::
+
+      You can figure you the production tag by looking at the footer of the
+      BEAT website. The corresponding tag name is found by prefixing a ``v``
+      before the version number. For example, the tag for version ``0.8.2`` of
+      the platform is ``v0.8.2``.
+
+
+   Also make sure to revert all dependent packages, so as to recreate the state
+   of the database schema as on the production site.
+
+
+4. Remove the current local development database so that the restore operation
+   can start from scratch::
+
+     [development]$ rm -rf django.sql3 web_dynamic_data
+
+
+5. Copy the backup tarball from the production server and restore it locally::
+
+     [development]$ scp root@beatweb:backups/<backup-filename>.tar.bz2
+     [development]$ ./bin/django restore <backup-filename>.tar.bz2
+
+   At this point, you have recreated a copy of your production system locally,
+   on your SQLite3 database.
+
+
+6. Reset queue configuration to allow for local running.
+
+   You may, optionally, reset the queue configuration of your installation so
+   that the environment you have is compatible with your development machine,
+   so that you can immediately run experiments locally. To do so, use the
+   ``qsetup`` Django command::
+
+     [development]$ ./bin/django qsetup --reset
+
+
+7. Apply migrations::
+
+   $ ./bin/django migrate
+
+
+At this point, you should have a complete development setup with all elements
+available on the production system installed locally. This system is fully
+capable of running experiments locally using your machine. Start a full system
+using ``localhost.py`` as explained on the localhost_ section above.
+
+
+Testing Django Migrations
+-------------------------
+
+Django migrations, introduced in version 1.7, is a useful feature for
+automatically migrating your database to new model schemas, if you get it
+right. Here is a recipe to make sure your migrations will work on your
+production system, allowing for quick and repetitive test/fix cycles.
+
+The key idea is that we follow the setup for the snapshot_ and then,
+locally backup our database and prefix so that we can quickly reproduce the
+migration test loop.
+
+
+1. Make sure you go through the snapshot_ instructions above (**up to
+   step 6 only**).
+
+
+2. Make a copy of the SQLite3 database::
+
+     $ cp -a django.sql3 django.sql3.backup
+
+   This backup will allow you to quickly test the migrations w/o having to
+   checkout the production version anymore.
+
+   Also, create a temporary git repository of ``web_dynamic_data``, so you can
+   cross-check changes and reset it in case of problems::
+
+     $ cd web_dynamic_data
+     $ git init .
+     $ git add .
+     $ git commit -m "Initial commit"
+     $ cd ..
+
+
+3. Go back to the HEAD or branch you were developping before::
+
+     $ git checkout HEAD
+
+
+4. Here is how to test/fix your migrations:
+
+   a. Run "django migrate"::
+
+        $ ./bin/django migrate
+
+   b. Check your database by visually inspecting it on the django web admin or
+      by manually dumping it.
+
+   c. If a problem is detected, fix it and revert the state::
+
+        $ cp -af django.sql3.backup django.sql3
+        $ cd web_dynamic_data && git reset --hard HEAD && git clean -fdx . & cd ..
+
+      .. note::
+
+         Tip: Write the above lines in a shell script so it is easy to repeat.
+
+      Go back to a. and restart.
+
+
+Javascript Management with Node.js/Bower
+----------------------------------------
+
+We manage javascript external packages with the help of Bower_. If you'd like
+to include more packages that will statically served with the Django web app,
+please consider including them at the appropriate section of ``buildout.cfg``.
+
+The included recipes will also download and install executables for
+``uglifyjs``, ``grunt``, ``csslint`` and ``jshint``, which can be useful for JS
+development.
+
+
+Issues
+------
+
+If you find problems concerning this package, please post a message to our
+`group mailing list`_. Currently open issues can be tracked at `our gitlab
+page`_.
+
+.. Place here references to all citations in lower case
+
+.. _django documentation: https://docs.djangoproject.com
+.. _pip: http://pypi.python.org/pypi/pip
+.. _easy_install: http://pypi.python.org/pypi/setuptools
+.. _zc.buildout: http://pypi.python.org/pypi/zc.buildout
+.. _virtualenv: http://pypi.python.org/pypi/virtualenv
+.. _group mailing list: https://groups.google.com/d/forum/beat-devel
+.. _our gitlab page: https://gitlab.idiap.ch/biometric/beat.web/issues
+.. _bower: http://bower.io

beat/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+__import__('pkg_resources').declare_namespace(__name__)

beat/web/.bowerrc

+{"directory": "static"}
\ No newline at end of file

beat/web/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #