Commit b4141d5e authored by André Anjos's avatar André Anjos 💬
Browse files

Open-source release

parents
Pipeline #892 passed with stage
*~
*.pyc
*.swp
*.swl
*.swm
*.swn
*.swo
parts/
*.egg-info/
develop-eggs/
eggs/
bin/
sphinx/
.installed.cfg
.mr.developer.cfg
.coverage
*.sql3
.DS_Store
beat/web/settings/settings.py
src/
web_dynamic_data*
.nfs*
beat/web/static/
doc/api/api/
html/
*.tar.bz2
py27-linux:
script:
- git clean -ffdx
- export TMPDIR=/var/tmp
- /idiap/project/beat/environments/staging/usr/bin/python bootstrap-buildout.py
- ./bin/buildout
- ./bin/python --version
- unset TMPDIR
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- export COVERAGE_FILE=.coverage.django
- ./bin/django test --settings=beat.web.settings.test -v 2
- export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
- export COVERAGE_FILE=.coverage.cmdline
- ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- ./bin/coverage combine
- ./bin/coverage report
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
- ./bin/sphinx-build doc/api html/api
- ./bin/sphinx-build doc/admin html/admin
- ./bin/sphinx-build doc/user html/user
tags:
- lidiap2015
py27-macosx:
script:
- git clean -ffdx
- /Users/buildbot/work/environments/beat/py27/bin/python bootstrap-buildout.py
- ./bin/buildout
- ./bin/python --version
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- export COVERAGE_FILE=.coverage.django
- ./bin/django test --settings=beat.web.settings.test -v 2
- export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
- export COVERAGE_FILE=.coverage.cmdline
- ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- ./bin/coverage combine
- ./bin/coverage report
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
- ./bin/sphinx-build doc/api html/api
- ./bin/sphinx-build doc/admin html/admin
- ./bin/sphinx-build doc/user html/user
tags:
- beat-macosx
.. vim: set fileencoding=utf-8 :
.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.web module of the BEAT platform. ..
.. ..
.. Commercial License Usage ..
.. Licensees holding valid commercial BEAT licenses may use this file in ..
.. accordance with the terms contained in a written agreement between you ..
.. and Idiap. For further information contact tto@idiap.ch ..
.. ..
.. Alternatively, this file may be used under the terms of the GNU Affero ..
.. Public License version 3 as published by the Free Software and appearing ..
.. in the file LICENSE.AGPL included in the packaging of this file. ..
.. The BEAT platform is distributed in the hope that it will be useful, but ..
.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
.. or FITNESS FOR A PARTICULAR PURPOSE. ..
.. ..
.. You should have received a copy of the GNU Affero Public License along ..
.. with the BEAT platform. If not, see http://www.gnu.org/licenses/. ..
===========================================================
Authors of the Biometrics Evaluation and Testing Platform
===========================================================
Andre Anjos <andre.anjos@idiap.ch>
Flavio Tarsetti <flavio.tarsetti@idiap.ch>
Laurent El-Shafey <laurent.el-shafey@idiap.ch>
Philip Abbet <philip.abbet@idiap.ch>
Samuel Gaist <samuel.gaist@idiap.ch>
Sebastien Marcel <sebastien.marcel@idiap.ch>
This diff is collapsed.
include LICENSE.AGPL README.rst bootstrap-buildout.py buildout.cfg
recursive-include beat/web *.css *.png *.gif *.svg *.ico *.js *.html *.rst *.txt
recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf
.. vim: set fileencoding=utf-8 :
.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.web module of the BEAT platform. ..
.. ..
.. Commercial License Usage ..
.. Licensees holding valid commercial BEAT licenses may use this file in ..
.. accordance with the terms contained in a written agreement between you ..
.. and Idiap. For further information contact tto@idiap.ch ..
.. ..
.. Alternatively, this file may be used under the terms of the GNU Affero ..
.. Public License version 3 as published by the Free Software and appearing ..
.. in the file LICENSE.AGPL included in the packaging of this file. ..
.. The BEAT platform is distributed in the hope that it will be useful, but ..
.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
.. or FITNESS FOR A PARTICULAR PURPOSE. ..
.. ..
.. You should have received a copy of the GNU Affero Public License along ..
.. with the BEAT platform. If not, see http://www.gnu.org/licenses/. ..
============================================
Biometrics Evaluation and Testing Platform
============================================
This package contains the source code for the web components of the BEAT
platform.
Installation
------------
Here is a recipe to get you started::
$ python bootstrap-buildout.py
$ ./bin/buildout
These 2 commands should download and install all non-installed dependencies and
get you a fully operational test and development environment.
.. note::
The python shell used in the first line of the previous command set
determines the python interpreter that will be used for all scripts developed
inside this package.
If you are on the Idiap filesystem, you may use
``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
package instead. It contains the same setup deployed at the final BEAT
machinery.
.. tip::
If you'd like to **speed-up** the installation, it is strongly advised you
prepare a preset virtual environment (see the virtualenv_ package) with all
required dependencies, so that ``./bin/buildout`` does not download and
installs all of them every time you cleanup. This technique should allow you
to quickly clean-up and re-start your working environment which is useful
during development.
In order to fetch currently needed dependencies, run::
$ ./bin/buildout #to setup once
$ ./bin/pip freeze > requirements.txt
Examine the file ``requirements.txt`` and remove packages you are either
developing locally (e.g., all that are under ``src``) or that you think you
don't need. The command ``pip freeze`` reports all installed packages and not
only those which are needed by your project. If the Python prompt you used
for bootstrapping already had a good set of packages installed, you may see
them there.
Once you have a satisfying ``requirements.txt`` file, you may proceed to
recreate a virtualenv_ you'll use for your development. Just call::
$ virtualenv ~/work/beat-env #--system-site-packages
To create the virtual environment. This new environment does not contain
system packages by default. You may override that by specifying
``--system-site-packages`` as suggested above. Then, install the required
packages on your new virtual environment::
$ ~/work/beat-env/bin/pip install -r requirements.txt
After that step is done, your virtual environment is ready for deployment.
You may now start from scratch to develop ``beat.web`` taking as base the
Python interpreter on your virtualenv_::
$ cd beat.web
$ git clean -fdx #full clean-up
$ ~/work/beat-env/bin/python bootstrap-buildout.py
$ ./bin/buildout
You'll realize the buildout step now takes considerably less time and you may
repeat this last step as much as needed. ``pip`` is a very flexible tool and
you may use it to manage the virtualenv_ installing and removing packages as
needed.
Documentation
-------------
Our documentation project is divided in 3 parts. The user guide is the only one
which is automatically built as part of the ``buildout`` procedure. The API and
administrators guide needs to be manually compiled if required.
To build the API documentation, just do::
$ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
$ ./bin/sphinx-build doc/api html/api
To build the administrator guide, just do::
$ ./bin/sphinx-build doc/admin html/admin
The above commands will build the stated guides, in HTML format, and dump
results into your local directory ``html``. You may navigate then to that
directory and, with your preferred web browser, open the file ``index.html`` to
browse the available documentation.
The basic user guide which includes information for users of the platform, is
built automatically upon ``buildout``. If you wish to build it and place it
alongside the other guides, you may do it as well like this::
$ ./bin/sphinx-build doc/user html/user
Instantiating a BEAT web server
-------------------------------
For a simple (development) web server, the default settings on
``beat/web/settings/settings.py`` should work out of the box. These settings:
* Instantiate the web service on the local host under port 8000 (the address
will be ``http://127.0.0.1:8000``
* Use an SQLITE3 database named ``django.sql3`` located on the current
working directory
* Run with full debug output
* It sets the working BEAT prefix to ``./prefix``
* A single user, called ``admin`` will be setup into the system
If you need to tweak these settings, just edit the file
``beat/web/settings/settings.py``. You may consult the `Django documentation`_
for detailed information on other settings.
Once the Django settings are in place, you can run a single command to fully
populate a development webserver::
$ ./bin/django install -v1
.. note::
Concerning databases installed by this command, we only explain the platform
how to **access** their data. It does not download the raw data for the
databases that you must procure yourself through the relevant web sites
(checkout the database pages on the Idiap instance of the BEAT platform for
details).
.. note::
If you need to specify your own path to the directories containing the
databases, you could just create a simple JSON file as follows::
{
"atnt/1": "/remote/databases/atnt",
"banca/2": "/remote/databases/banca"
}
Then just use the previous script with the option ``--database-root-file``::
$ ./bin/django install -v1 --database-root-file=MYFILE.json
By default, paths to the root of all databases are set to match the Idiap
Research Institute filesystem organisation.
.. note::
For every installed database, you'll need to generate their data indices,
which allows the platform to correctly parallelize algorithms. To do so, for
every combination of database and version you wish to support, run the
following command::
$ ./bin/beat -p web_dynamic_data db index <name>/<version>
Replacing the strings ``<name>`` by the name of the database you wish to dump
the indices for, together with the version in ``<version>``. For example, to
dump the indices for the AT&T database, version 1, do the following::
$ ./bin/beat -p web_dynamic_data db index atnt/1
Once the contributions and users are in place, you're ready to start the test
server::
$ ./bin/django runserver -v3
At this point, your platform can be accessed by typing the URL
``http://127.0.0.1:8000`` in a web browser on the machine the server is
running.
.. _localhost:
Localhost
---------
To effectively use your new server and test all aspects of it, you'll also need
a scheduler with at least one attached worker that can execute experiments. For
most development purposes, a simple 3-node system, with all components running
on the current (local) host is sufficient.
Here is a recipe to start a simple 3-node system in which the local worker uses
the system-wide installed Python interpreter to execute the algorithms.
First, make sure the program ``cpulimit`` is available on your system. The BEAT
platform uses this program to control slot usage on the scheduling/worker
level::
$ cpulimit -h
If that is not the case, then you need to install it. Either install a package
that is native to your system (e.g. on Debian or Ubuntu platforms) or compile
the checked-out version available at ``src/cpulimit``::
$ cd src/cpulimit;
$ make
$ ./src/cpulimit -h #to test it
$ cd ../../bin #go back to the root of beat.web and the into the `bin' dir
$ ln -s ../src/cpulimit/src/cpulimit
$ cd .. #go back to the root of beat.web
Now start the localhost system::
$ ./bin/localhost.py -v
...
You may inspect this programs help message for details on its usage and
options.
Once the localhost system is started and the scheduler is properly configured,
you may open a browser window to your `localhost, port 8000
<http://127.0.0.1:8000>`_, to get started with your locally installed platform.
Localhost with ``DEBUG=False``
==============================
If you need to test the RESTful API, it is better to do it without Django
throwing you HTML error pages. For that, you'll need to start the Django
development server with slightly different settings::
$ ./bin/localhost.py -v --settings=beat.web.settings.nodebug
Triggering a Scheduler Reconfiguration
======================================
If you modify the queue configuration on the Django administrative panel,
you'll need to notify the scheduler of those changes. You can trigger a
scheduler (hot) re-configuration using the following command-line program::
$ ./bin/django qconf
.. note::
Optionally, you may also visit `your local scheduler page
<http://127.0.0.1:8000/backend/scheduler>`, and hit the (green) button that
says "Send configuration to Scheduler". It has the same effect.
Unit Testing
------------
After installation, it is possible to run our suite of unit tests. To do so,
use::
$ ./bin/django test --settings=beat.web.settings.test -v 2
You may pass filtering criteria to just launch tests for a particular set of
``beat.web`` applications. For example, to run tests only concerning
``beat.web.toolchains``, run::
$ ./bin/django test --settings=beat.web.settings.test -v 2 beat.web.toolchains.tests
To measure coverage, you must set an environment variable for nose::
$ NOSE_WITH_COVERAGE=1 NOSE_COVER_PACKAGE=beat.web ./bin/django test --settings=beat.web.settings.test -v 2
.. _snapshot:
Local Development Server
------------------------
It is easy to quickly setup a local system for development, taking as base the
current state of a production system.
1. Before starting, make sure you have gone through, at least once, over the
localhost_ instructions above. It explains the very basic setup required for
a complete development environment.
2. Dump and back-up your current **production** BEAT database::
[production]$ ./bin/django backup
3. [Optional] If you have made important modifications between the contents
available at your production server and your currently checked-out source,
you'll need to run Django migrations on data imported from the production
server. If you need to do this, make sure you don't have unapplied commits
to your local **development** package and reset it to the production tag::
[development]$ git checkout <production-tag>
.. note::
You can figure you the production tag by looking at the footer of the
BEAT website. The corresponding tag name is found by prefixing a ``v``
before the version number. For example, the tag for version ``0.8.2`` of
the platform is ``v0.8.2``.
Also make sure to revert all dependent packages, so as to recreate the state
of the database schema as on the production site.
4. Remove the current local development database so that the restore operation
can start from scratch::
[development]$ rm -rf django.sql3 web_dynamic_data
5. Copy the backup tarball from the production server and restore it locally::
[development]$ scp root@beatweb:backups/<backup-filename>.tar.bz2
[development]$ ./bin/django restore <backup-filename>.tar.bz2
At this point, you have recreated a copy of your production system locally,
on your SQLite3 database.
6. Reset queue configuration to allow for local running.
You may, optionally, reset the queue configuration of your installation so
that the environment you have is compatible with your development machine,
so that you can immediately run experiments locally. To do so, use the
``qsetup`` Django command::
[development]$ ./bin/django qsetup --reset
7. Apply migrations::
$ ./bin/django migrate
At this point, you should have a complete development setup with all elements
available on the production system installed locally. This system is fully
capable of running experiments locally using your machine. Start a full system
using ``localhost.py`` as explained on the localhost_ section above.
Testing Django Migrations
-------------------------
Django migrations, introduced in version 1.7, is a useful feature for
automatically migrating your database to new model schemas, if you get it
right. Here is a recipe to make sure your migrations will work on your
production system, allowing for quick and repetitive test/fix cycles.
The key idea is that we follow the setup for the snapshot_ and then,
locally backup our database and prefix so that we can quickly reproduce the
migration test loop.
1. Make sure you go through the snapshot_ instructions above (**up to
step 6 only**).
2. Make a copy of the SQLite3 database::
$ cp -a django.sql3 django.sql3.backup
This backup will allow you to quickly test the migrations w/o having to
checkout the production version anymore.
Also, create a temporary git repository of ``web_dynamic_data``, so you can
cross-check changes and reset it in case of problems::
$ cd web_dynamic_data
$ git init .
$ git add .
$ git commit -m "Initial commit"
$ cd ..
3. Go back to the HEAD or branch you were developping before::
$ git checkout HEAD
4. Here is how to test/fix your migrations:
a. Run "django migrate"::
$ ./bin/django migrate
b. Check your database by visually inspecting it on the django web admin or
by manually dumping it.
c. If a problem is detected, fix it and revert the state::
$ cp -af django.sql3.backup django.sql3
$ cd web_dynamic_data && git reset --hard HEAD && git clean -fdx . & cd ..
.. note::
Tip: Write the above lines in a shell script so it is easy to repeat.
Go back to a. and restart.
Javascript Management with Node.js/Bower
----------------------------------------
We manage javascript external packages with the help of Bower_. If you'd like
to include more packages that will statically served with the Django web app,
please consider including them at the appropriate section of ``buildout.cfg``.
The included recipes will also download and install executables for
``uglifyjs``, ``grunt``, ``csslint`` and ``jshint``, which can be useful for JS
development.
Issues
------
If you find problems concerning this package, please post a message to our
`group mailing list`_. Currently open issues can be tracked at `our gitlab
page`_.
.. Place here references to all citations in lower case
.. _django documentation: https://docs.djangoproject.com
.. _pip: http://pypi.python.org/pypi/pip
.. _easy_install: http://pypi.python.org/pypi/setuptools
.. _zc.buildout: http://pypi.python.org/pypi/zc.buildout
.. _virtualenv: http://pypi.python.org/pypi/virtualenv
.. _group mailing list: https://groups.google.com/d/forum/beat-devel
.. _our gitlab page: https://gitlab.idiap.ch/biometric/beat.web/issues
.. _bower: http://bower.io
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
{"directory": "static"}
\ No newline at end of file
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #