Commit a0d21253 authored by Manuel Günther's avatar Manuel Günther
Browse files

Added tons of documentation

parent 9928bdc8
......@@ -3,3 +3,6 @@ from .PCA import PCA
from .LDA import LDA
from .PLDA import PLDA
from .BIC import BIC
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <>
"""Basic features for biometric recognition"""
from .Extractor import Extractor
from .Linearize import Linearize
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -106,3 +106,6 @@ class Grid:
def is_local(self):
"""Returns whether this grid setup should use the local submission or the SGE grid."""
return self.grid_type == 'local'
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
from .Preprocessor import Preprocessor
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -39,7 +39,7 @@ def resources():
def databases():
import argparse
database_replacement = "/idiap/home/%s/.bob_bio_databases.txt" % os.environ["USER"] if os.path.isdir("/idiap") else "/home/%s/.bob_bio_databases.txt" % os.environ["USER"]
database_replacement = "%s/.bob_bio_databases.txt" % os.environ["HOME"]
parser = argparse.ArgumentParser(description="Prints a list of directories for registered databases", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-D', '--database-directories-file', metavar = 'FILE', default = database_replacement, help = 'The file, where database directories are stored (to avoid changing the database configurations)')
......@@ -5,3 +5,6 @@ from .algorithm import *
from .scoring import *
from .command_line import *
from .grid import *
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -5,7 +5,7 @@ import sys
import bob.core
logger = bob.core.log.setup("")
from ..utils import load_resource, resource_keys
from .. import utils
from . import FileSelector
"""Execute biometric recognition algorithms on a certain biometric database.
......@@ -23,13 +23,13 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
############## options that are required to be specified #######################
config_group = parser.add_argument_group('\nParameters defining the experiment. Most of these parameters can be a registered resource, a configuration file, or even a string that defines a newly created object')
config_group.add_argument('-d', '--database', metavar = 'x', nargs = '+', required = True,
help = 'Database and the protocol; registered databases are: %s' % resource_keys('database', exclude_resources_from))
help = 'Database and the protocol; registered databases are: %s' % utils.resource_keys('database', exclude_resources_from))
config_group.add_argument('-p', '--preprocessor', metavar = 'x', nargs = '+', required = True,
help = 'Data preprocessing; registered preprocessors are: %s' % resource_keys('preprocessor', exclude_resources_from))
help = 'Data preprocessing; registered preprocessors are: %s' % utils.resource_keys('preprocessor', exclude_resources_from))
config_group.add_argument('-e', '--extractor', metavar = 'x', nargs = '+', required = True,
help = 'Feature extraction; registered feature extractors are: %s' % resource_keys('extractor', exclude_resources_from))
help = 'Feature extraction; registered feature extractors are: %s' % utils.resource_keys('extractor', exclude_resources_from))
config_group.add_argument('-a', '--algorithm', metavar = 'x', nargs = '+', required = True,
help = 'Biometric recognition; registered algorithms are: %s' % resource_keys('algorithm', exclude_resources_from))
help = 'Biometric recognition; registered algorithms are: %s' % utils.resource_keys('algorithm', exclude_resources_from))
config_group.add_argument('-g', '--grid', metavar = 'x', nargs = '+',
help = 'Configuration for the grid setup; if not specified, the commands are executed sequentially on the local machine.')
config_group.add_argument('--imports', metavar = 'LIB', nargs = '+', default = [''],
......@@ -48,7 +48,7 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
is_idiap = os.path.isdir("/idiap")
temp = "/idiap/temp/%s/database-name/sub-directory" % os.environ["USER"] if is_idiap else "temp"
results = "/idiap/user/%s/database-name/sub-directory" % os.environ["USER"] if is_idiap else "results"
database_replacement = "/idiap/home/%s/.bob_bio_databases.txt" % os.environ["USER"] if is_idiap else "/home/%s/.bob_bio_databases.txt" % os.environ["USER"]
database_replacement = "%s/.bob_bio_databases.txt" % os.environ["HOME"]
dir_group = parser.add_argument_group('\nDirectories that can be changed according to your requirements')
dir_group.add_argument('-T', '--temp-directory', metavar = 'DIR',
......@@ -150,12 +150,12 @@ def initialize(parsers, command_line_parameters = None, skips = []):
args.timer = ('real', 'system', 'user')
# load configuration resources
args.database = load_resource(' '.join(args.database), 'database', imports = args.imports)
args.preprocessor = load_resource(' '.join(args.preprocessor), 'preprocessor', imports = args.imports)
args.extractor = load_resource(' '.join(args.extractor), 'extractor', imports = args.imports)
args.algorithm = load_resource(' '.join(args.algorithm), 'algorithm', imports = args.imports)
args.database = utils.load_resource(' '.join(args.database), 'database', imports = args.imports)
args.preprocessor = utils.load_resource(' '.join(args.preprocessor), 'preprocessor', imports = args.imports)
args.extractor = utils.load_resource(' '.join(args.extractor), 'extractor', imports = args.imports)
args.algorithm = utils.load_resource(' '.join(args.algorithm), 'algorithm', imports = args.imports)
if args.grid is not None:
args.grid = load_resource(' '.join(args.grid), 'grid', imports = args.imports)
args.grid = utils.load_resource(' '.join(args.grid), 'grid', imports = args.imports)
# set base directories
is_idiap = os.path.isdir("/idiap")
......@@ -237,8 +237,8 @@ def write_info(args, command_line_parameters, executable):
f.write(command_line([executable] + command_line_parameters) + "\n\n")
f.write("Database:\n%s\n\n" % args.database)
f.write("Preprocessing:\n%s\n\n" % args.preprocessor)
f.write("Feature Extraction:\n%s\n\n" % args.extractor)
f.write("Preprocessor:\n%s\n\n" % args.preprocessor)
f.write("Extractor:\n%s\n\n" % args.extractor)
f.write("Algorithm:\n%s\n\n" % args.algorithm)
except IOError:
logger.error("Could not write the experimental setup into file '%s'", args.info_file)
......@@ -3,7 +3,7 @@ from __future__ import print_function
import sys
import os
import math
from ..grid import Grid
from .. import grid
from .command_line import command_line
import bob.core
......@@ -48,7 +48,7 @@ class GridSubmission:
self.executable = executable
if args.grid is not None:
assert isinstance(args.grid, Grid)
assert isinstance(args.grid, grid.Grid)
# find, where jman is installed
jmans = bob.extension.find_executable('jman', prefixes = ['bin'])
......@@ -174,7 +174,7 @@ def list_resources(keyword, strip=['dummy']):
entry_points = _get_entry_points(keyword, strip)
last_dist = None
retval = ""
length = max(len( for entry_point in entry_points)
length = max(len( for entry_point in entry_points) if entry_points else 1
for entry_point in sorted(entry_points):
if last_dist != str(entry_point.dist):
......@@ -198,7 +198,6 @@ def database_directories(strip=['dummy'], replacements = None):
db = load_resource(, 'database')
dirs[] = [db.original_directory]
# import ipdb; ipdb.set_trace()
if db.annotation_directory is not None:
except (AttributeError, ValueError):
......@@ -86,7 +86,7 @@ release = distribution.version
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
#exclude_patterns = ['**/links.rst']
exclude_patterns = ['links.rst']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
.. vim: set fileencoding=utf-8 :
.. author: Manuel Günther <>
.. date: Thu Sep 20 11:58:57 CEST 2012
.. _experiments:
Running Biometric Recognition Experiments
Now, you are almost ready to run your first biometric recognition experiment.
Just a little bit of theory, and then: off we go.
Structure of a Biometric Recognition Experiment
Each biometric recognition experiment that is run with ```` is divided into several steps.
The steps are:
1. Data preprocessing: Raw data is preprocessed, e.g., for face recognition, faces are detected, images are aligned and photometrically enhanced.
2. Feature extractor training: Feature extraction parameters are learned.
3. Feature extraction: Features are extracted from the preprocessed data.
4. Feature projector training: Parameters of a subspace-projection of the features are learned.
5. Feature projection: The extracted features are projected into a subspace.
6. Model enroller training: The ways how to enroll models from extracted or projected features is learned.
7. Model enrollment: One model is enrolled from the features of one or more images.
8. Scoring: The verification scores between various models and probe features are computed.
9. Evaluation: The computed scores are evaluated and curves are plotted.
These 9 steps are divided into four distinct groups, which are discussed in more detail later:
* Preprocessing (only step 1)
* Feature extraction (steps 2 and 3)
* Biometric recognition (steps 4 to 8)
* Evaluation (step 9)
The communication between two steps is file-based, usually using a binary HDF5_ interface, which is implemented in the :py:class:`` class.
The output of one step usually serves as the input of the subsequent step(s).
Depending on the algorithm, some of the steps are not applicable/available.
E.g. most of the feature extractors do not need a special training step, or some algorithms do not require a subspace projection.
In these cases, the according steps are skipped.
```` takes care that always the correct files are forwarded to the subsequent steps.
Running Experiments
To run an experiment, we provide a generic script ``./bin/``, which is highly parametrizable.
To get a complete list of command line options, please run:
.. code-block:: sh
$ ./bin/ --help
Whoops, that's a lot of options.
But, no worries, most of them have proper default values.
.. note::
Sometimes, command line options have a long version starting with ``--`` and a short one starting with a single ``-``.
In this section, only the long names of the arguments are listed, please refer to ``./bin/ --help`` (or short: ``./bin/ -h``) for the abbreviations.
There are five command line options, which are required and sufficient to define the complete biometric recognition experiment.
These five options are:
* ``--database``: The database to run the experiments on
* ``--preprocessor``: The data preprocessor
* ``--extractor``: The feature extractor
* ``--algorithm``: The recognition algorithm
* ``--sub-directory``: A descriptive name for your experiment, which will serve as a sub-directory
The first four parameters, i.e., the ``database``, the ``preprocessor``, the ``extractor`` and the ``algorithm`` can be specified in several different ways.
For the start, we will use only the registered :ref:`Resources <resources>`.
These resources define the source code that will be used to compute the experiments, as well as all the meta-parameters of the algorithms (which we will call the **configuration**).
To get a list of registered resources, please call:
.. code-block:: sh
$ ./bin/
Each package in ```` defines its own resources, and the printed list of registered resources differs according to the installed packages.
If only ```` is installed, no databases and no preprocessors will be listed.
.. note::
You will also find some ``grid`` resources being listed.
These type of resources will be explained :ref:`later <grid>`.
One command line option, which is not required, but recommended, is the ``--verbose`` option.
By default, the algorithms are set up to execute quietly, and only errors are reported.
To change this behavior, you can use the ``--verbose`` option several times to increase the verbosity level to show:
1) Warning messages
2) Informative messages
3) Debug messages
When running experiments, my personal preference is verbose level 2, which can be enabled by ``--verbose --verbose``, or using the short version: ``-vv``.
So, a typical biometric recognition experiment (in this case, face recognition) could look something like:
.. code-block:: sh
$ ./bin/ --database mobio-image --preprocessor face-crop-eyes --extractor linearize --algorithm pca --sub-directory pca-experiment -vv
.. note::
To be able to run exactly the command line from above, it requires to have :ref:` <>` installed.
Before running an experiment, it is recommended to add the ``--dry-run`` option, so that it will only print, which steps would be executed, without actually executing them, and make sure that everything works as expected.
The final result of the experiment will be one (or more) score file(s).
Usually, they will be called something like ``scores-dev``.
By default, you can find them in a sub-directory the ``result`` directory, but you can change this option using the ``--result-directory`` command line option.
.. note::
At Idiap_, the default result directory differs, see ``./bin/ --help`` for your directory.
Evaluating Experiments
After the experiment has finished successfully, one or more text file containing all the scores are written.
To evaluate the experiment, you can use the generic ``./bin/`` script, which has properties for all prevalent evaluation types, such as CMC, ROC and DET plots, as well as computing recognition rates, EER/HTER, Cllr and minDCF.
Additionally, a combination of different algorithms can be plotted into the same files.
Just specify all the score files that you want to evaluate using the ``--dev-files`` option, and possible legends for the plots (in the same order) using the ``--legends`` option, and the according plots will be generated.
For example, to create a ROC curve for the experiment above, use:
.. code-block:: sh
$ ./bin/ --dev-files results/pca-experiment/male/nonorm/scores-dev --legend MOBIO --roc MOBIO_MALE_ROC.pdf -vv
Please note that there exists another file called ```` inside the result directory.
This file is a pure text file and contains the complete configuration of the experiment.
With this configuration it is possible to inspect all default parameters of the algorithms, and even to re-run the exact same experiment.
Running in Parallel
One important property of the ``./bin/`` script is that it can run in parallel, using either several threads on the local machine, or an SGE grid.
To achieve that, ```` is well-integrated with our SGE grid toolkit GridTK_, which we have selected as a python package in the :ref:`Installation <installation>` section.
The ``./bin/`` script can submit jobs either to the SGE grid, or to a local scheduler, keeping track of dependencies between the jobs.
The GridTK_ keeps a list of jobs in a local database, which by default is called ``submitted.sql3``, but which can be overwritten with the ``--gridtk-database-file`` option.
Please refer to the `GridTK documentation <>`_ for more details on how to use the Job Manager ``./bin/jman``.
Two different types of ``grid`` resources are defined, which can be used with the ``--grid`` command line option.
The first type of resources will submit jobs to an SGE grid.
They are mainly designed to run in the Idiap_ SGE grid and might need some adaptations to run on your grid.
The second type of resources will submit jobs to a local queue, which needs to be run by hand (e.g., using ``./bin/jman --local run-scheduler --parallel 4``), or by using the command line option ``--run-local-scheduler``.
The difference between the two types of resources is that the local submission usually starts with ``local-``, while the SGE resource does not.
Hence, to run the same experiment as above using four parallel threads on the local machine, re-nicing the jobs to level 10, simply call:
.. code-block:: sh
$ ./bin/ --database mobio-image --preprocessor face-crop-eyes --extractor linearize --algorithm pca --sub-directory pca-experiment -vv --grid local-p4 --run-local-scheduler --nice 10
.. note::
You might realize that the second execution of the same experiment is much faster than the first one.
This is due to the fact that those parts of the experiment, which have been successfully executed before (i.e., the according files already exist), are skipped.
To override this behavior, i.e., to always regenerate all parts of the experiments, you can use the ``--force`` option.
Command Line Options to change Default Behavior
Additionally to the required command line arguments discussed above, there are several options to modify the behavior of the experiments.
One set of command line options change the directory structure of the output.
By default, intermediate (temporary) files are by default written to the ``temp`` directory, which can be overridden by the ``--temp-directory`` command line option, which expects relative or absolute paths:
Re-using Parts of Experiments
If you want to re-use parts previous experiments, you can specify the directories (which are relative to the ``--temp-directory``, but you can also specify absolute paths):
* ``--preprocessed-data-directory``
* ``--extracted-directory``
* ``--projected-directory``
* ``--models-directories`` (one for each the models and the ZT-norm-models, see below)
or even trained extractor, projector, or enroller (i.e., the results of the extractor, projector, or enroller training):
* ``--extractor-file``
* ``--projector-file``
* ``--enroller-file``
For that purpose, it is also useful to skip parts of the tool chain.
To do that you can use:
* ``--skip-preprocessing``
* ``--skip-extractor-training``
* ``--skip-extraction``
* ``--skip-projector-training``
* ``--skip-projection``
* ``--skip-enroller-training``
* ``--skip-enrollment``
* ``--skip-score-computation``
* ``--skip-concatenation``
* ``--skip-calibration``
although by default files that already exist are not re-created.
You can use the ``--force`` argument combined with the ``--skip...`` arguments (in which case the skip is preferred).
To run just a sub-selection of the tool chain, you can also use the ``--execute-only`` option, which takes a list of options out of: ``preprocessing``, ``extractor-training``, ``extraction``, ``projector-training``, ``projection``, ``enroller-training``, ``enrollment``, ``score-computation``, ``concatenation`` or ``calibration``.
Database-dependent Arguments
Many databases define several protocols that can be executed.
To change the protocol, you can either modify the configuration file, or simply use the ``--protocol`` option.
Some databases define several kinds of evaluation setups.
For example, often two groups of data are defined, a so-called *development set* and an *evaluation set*.
The scores of the two groups will be concatenated into two files called **scores-dev** and **scores-eval**, which are located in the score directory (see above).
In this case, by default only the development set is employed.
To use both groups, just specify ``--groups dev eval`` (of course, you can also only use the ``'eval'`` set by calling ``--groups eval``).
One score normalization technique is the so-called ZT score normalization.
To enable this, simply use the ``--zt-norm`` option.
If the ZT-norm is enabled, two sets of scores will be computed, and they will be placed in two different sub-directories of the score directory, which are by default called **nonorm** and **ztnorm**, but which can be changed using the ``--zt-score-directories`` option.
Other Arguments
For some applications it is interesting to get calibrated scores.
Simply add the ``--calibrate-scores`` option and another set of score files will be created by training the score calibration on the scores of the ``'dev'`` group and execute it to all available groups.
The scores will be located at the same directory as the **nonorm** and **ztnorm** scores, and the file names are **calibrated-dev** (and **calibrated-eval** if applicable) .
.. include:: links.rst
This diff is collapsed.
Tools implemented in
.. automodule::
.. automodule::
.. automodule::
.. automodule::
Grid Configuration
.. automodule::
.. include:: links.rst
.. vim: set fileencoding=utf-8 :
.. Andre Anjos <>
.. Mon 13 Aug 2012 12:36:40 CEST
.. author: Manuel Günther <>
.. date: Thu Sep 20 11:58:57 CEST 2012
Bob Example Project
Running Biometric Recognition Experiments
Package Documentation
The ```` packages provide open source tools to run comparable and reproducible biometric recognition experiments.
To design a biometric recognition experiment, one has to choose:
.. automodule::
* a databases containing the original data, and a protocol that defines how to use the data,
* a data preprocessing algorithm, i.e., face detection for face recognition experiments or voice activity detection for speaker recognition
* the type of features to extract from the preprocessed data,
* the biometric recognition algorithm to employ, and
* the way to evaluate the results
For any of these parts, several different types are implemented in the ```` packages, and basically any combination of the five parts can be executed.
For each type, several meta-parameters can be tested.
This results in a nearly infinite amount of possible experiments that can be run using the current setup.
But it is also possible to use your own database, preprocessing, feature type, or biometric recognition algorithm and test this against the baseline algorithms implemented in the our packages.
.. automodule::
The ```` packages derived from the former `FaceRecLib <>`__, which is herewith outdated.
This package :py:mod:`` includes the basic definition of a biometric recognition experiment, as well as a generic script, which can execute the full biometric experiment in a single command line.
Changing the employed tolls such as the database, protocol, preprocessor, feature extractor or recognition algorithm is as simple as changing a command line parameter.
The implementation of (most of) the tools is separated into other packages in the ```` namespace.
All these packages can be easily combined.
Here is a growing list of derived packages:
* :ref:` <>` Tools to run speaker recognition experiments, including voice activity detection, Cepstral feature extraction, and speaker databases
* :ref:` <>` Tools to run face recognition experiments, such as face detection, facial feature extraction and comparison, and face image databases
* :ref:` <>` An extension of face recognition algorithms to run on video data, and the according video databases
* :ref:` <>` Algorithms based on Gaussian Mixture Modeling (GMM) such as Inter-Session Variability modeling (ISV) or Total Variability modeling (TV, aka. I-Vector)
* :ref:` <>` Wrapper classes for the `CSU Face Recognition Resources <>`_ to be run with ````.
If you are interested, please continue reading:
Users Guide
.. toctree::
:maxdepth: 2
.. evaluate
Reference Manual
.. toctree::
:maxdepth: 2
This documentation is still under development.
Here is a list of things that needs to be done:
.. todolist::
Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
.. include:: links.rst
.. vim: set fileencoding=utf-8 :
.. author: Manuel Günther <>
.. date: Thu Sep 20 11:58:57 CEST 2012
.. _installation:
Installation Instructions
As noted before, this package is part of the ```` packages, which in turn are part of the signal-processing and machine learning toolbox Bob_.
To install `Packages of Bob <>`_, please read the `Installation Instructions <>`_.
For Bob_ to be able to work properly, some dependent packages are required to be installed.
Please make sure that you have read the `Dependencies <>`_ for your operating system.
.. note::
Currently, running Bob_ under MS Windows in not yet supported.
However, we found that running Bob_ in a virtual Unix environment such as the one provided by VirtualBox_ is a good alternative.
The most simple and most convenient way to use the ```` tools is to use a ``zc.buildout`` package, as explained in more detail `here <>`__.
There, in the ``eggs`` section of the ``buildout.cfg`` file, simply list the ```` packages that you want, like:
.. code-block:: python
eggs =
in order to download and install all packages that are required for your experiments.
In the example above, you might want to run a video face recognition experiments using the :py:class:`` and the :py:class:`` feature extractor defined in :ref:` <>`, the :py:class:`` algorithm defined in :ref:` <>` and the video extensions defined in :ref:` <>`, using the YouTube faces database interface defined in :ref:` <>`.
Running the simple command line:
.. code-block:: sh
$ python
$ ./bin/buildout
will the download and install all dependent packages locally (relative to your current working directory), and create a ``./bin`` directory containing all the necessary scripts to run the experiments.
With ```` you will run biometric recognition experiments using some default biometric recognition databases.
Though the verification protocols are implemented in ````, the original data are **not included**.
To download the original data of the databases, please refer to the according Web-pages.
Database URL's will be given in the :ref:`databases` section.
After downloading the original data for the databases, you will need to tell ````, where these databases can be found.
For this purpose, we have decided to implement a special file, where you can set your directories.
By default, this file is located in ``~/.bob_bio_databases.txt``, and it contains several lines, each line looking somewhat like:
.. code-block:: text
[YOUR_ATNT_DATABASE_DIRECTORY] = /path/to/your/directory
.. note::
If this file does not exist, feel free to create and populate it yourself.
Please use ``./bin/`` for a list of known databases, where you can see the raw ``[YOUR_DATABASE_PATH]`` entries for all databases that you haven't updated, and the corrected paths for those you have.
.. note::
If you have installed only ````, there is no database listed -- as all databases are included in other packages, such as :ref:` <>` or :ref:` <>`.
Test your Installation
One of the scripts that were generated during the bootstrap/buildout step is a test script.
To verify your installation, you should run the script running the nose tests for each of the ```` packages:
.. code-block:: sh
$ ./bin/nosetests -vs
$ ./bin/nosetests -vs
Some of the tests that are run require the images of the `AT&T database`_ database.
If the database is not found on your system, it will automatically download and extract the `AT&T database`_ a temporary directory, **which will not be erased**.
To avoid the download to happen each time you call the nose tests, please:
1. Download the `AT&T database`_ database and extract it to the directory of your choice.
2. Set an environment variable ``ATNT_DATABASE_DIRECTORY`` to the directory, where you extracted the database to.
For example, in a ``bash`` you can call:
.. code-block:: sh
$ export ATNT_DATABASE_DIRECTORY=/path/to/your/copy/of/atnt
.. note::
To set the directory permanently, you can also change the ``atnt_default_directory`` in the file `bob/bio/base/test/ <file:../bob/bio/base/test/>`_.
In this case, there is no need to set the environment variable any more.
In case any of the tests fail for unexplainable reasons, please file a bug report through the `GitHub bug reporting system`_.
.. note::