Commit 79d8bc26 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'creating-baselines' into 'master'

Created the Baselines Concept

See merge request !151
parents 8f48d57f 8b4068a2
Pipeline #20229 canceled with stage
......@@ -6,6 +6,7 @@ from . import algorithm
from . import tools
from . import grid # only one file, not complete directory
from . import annotator
from . import baseline
from . import script
from . import test
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <>
from .. import resource_keys, load_resource
def search_preprocessor(db_name, keys):
Wrapper that searches for preprocessors for specific databases.
If not found, the default preprocessor is returned
for k in keys:
if db_name.startswith(k):
return k
return "default"
def get_available_databases():
Get all the available databases through the database entry-points
available_databases = dict()
all_databases = resource_keys('database', strip=[])
for database in all_databases:
database_entry_point = load_resource(database, 'database')
available_databases[database] = dict()
# Checking if the database has data for the ZT normalization
available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
available_databases[database]["groups"] = []
# Searching for database groups
groups = list(database_entry_point.groups())
for g in ["dev", "eval"]:
available_databases[database]["groups"] += [g] if g in groups else []
except Exception:
# In case the method groups is not implemented
available_databases[database]["groups"] = ["dev"]
except Exception:
return available_databases
class Baseline(object):
Base class to define baselines
A Baseline is composed by the triplet
:any:``, and
name : str
Name of the baseline. This name will be displayed in the command line
preprocessors : dict
Dictionary containing all possible preprocessors
extractor : str
Registered resource or a config file containing the feature extractor
algorithm : str
Registered resource or a config file containing the algorithm
def __init__(self, name, preprocessors, extractor, algorithm, **kwargs):
super(Baseline, self).__init__(**kwargs) = name
self.preprocessors = preprocessors
self.extractor = extractor
self.algorithm = algorithm
from .Baseline import Baseline, search_preprocessor, get_available_databases
def get_config():
"""Returns a string containing the configuration information.
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened. Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
for obj in args:
obj.__module__ = __name__
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <>
A script to run biometric recognition baselines
from .. import load_resource
import os
from .verify import main as verify
from ..baseline import get_available_databases, search_preprocessor
from bob.extension.scripts.click_helper import verbosity_option
import click
@click.command(context_settings={'ignore_unknown_options': True,
'allow_extra_args': True})
@click.argument('baseline', required=True)
@click.argument('database', required=True)
def baseline(ctx, baseline, database):
"""Run a biometric recognition baseline.
$ bob bio baseline eigenface atnt -vvv
which will run the eigenface baseline (from on the atnt
Check out all baselines available by running:
` --types baseline`
and all available databases by running:
` --types database`
This script accepts parameters accepted by as well.
See ` --help` for the extra options that you can pass.
Hint: pass `--grid demanding` to run the baseline on the SGE grid.
Hint: pass `--temp-directory <dir>` to set the directory for temporary files
Hint: pass `--result-directory <dir>` to set the directory for resulting score files
# Triggering training for each baseline/database
loaded_baseline = load_resource(
baseline, 'baseline', package_prefix="")
# this is the default sub-directory that is used
sub_directory = os.path.join(database, baseline)
# find the compatible preprocessor for this database
database_data = get_available_databases()[database]
db = search_preprocessor(database, loaded_baseline.preprocessors.keys())
preprocessor = loaded_baseline.preprocessors[db]
# call verify with all parameters
parameters = [
'-p', preprocessor,
'-e', loaded_baseline.extractor,
'-d', database,
'-a', loaded_baseline.algorithm,
'--sub-directory', sub_directory
] + ['-v'] * ctx.meta['verbosity']
parameters += ['--groups'] + database_data["groups"]
verify(parameters + ctx.args)
......@@ -9,8 +9,8 @@ def resources(command_line_parameters = None):
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--types", '-t', nargs = '+',
choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator'),
default = ('d', 'p', 'e', 'a', 'g', 'c', 'an'),
choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator', 'b', 'baseline'),
default = ('d', 'p', 'e', 'a', 'g', 'c', 'an', 'b'),
help = "Select the resource types that should be listed.")
parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources")
......@@ -55,6 +55,10 @@ def resources(command_line_parameters = None):
print ("\nList of registered annotators:")
print ('annotator', **kwargs))
if 'b' in args.types or 'baseline' in args.types:
print ("\nList of registered baseline:")
print ('baseline', **kwargs))
def databases(command_line_parameters = None):
from import Baseline
import pkg_resources
import os
dummy_dir = pkg_resources.resource_filename('', 'test/dummy')
baseline = Baseline(name="dummy",
preprocessors={"default": os.path.join(dummy_dir, '')},
extractor=os.path.join(dummy_dir, ''),
algorithm=os.path.join(dummy_dir, ''))
import tempfile
import shutil
from click.testing import CliRunner
from import baseline
def test_baselines():
tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
runner = CliRunner()
result = runner.invoke(baseline, args=('dummy', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
assertion_error_message = (
'Command exited with this output: `{}\' \n'
'If the output is empty, you can run this script locally to see '
'what is wrong:\n'
'bin/bob bio baseline -d dummy -a dummy -o /tmp/temp_annotations'
assert result.exit_code == 0, assertion_error_message
......@@ -21,7 +21,7 @@ logger = logging.getLogger("")
#: Keywords for which resources are defined.
valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator')
valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator', 'baseline')
def _collect_config(paths):
Defining baselines
Once you have a biometric system well established, tuned and working for a
particular database (or a particular set of databases), you may want to provide
**an easier to reproduce** way to share it. For this purpose, we defined
something called baseline.
A baseline (:any:``) is composed by the triplet
of :any:``,
:any:`` and
First, check it out the baselines ready to be triggered in your environment by
.. code-block:: sh
$ bob bio baseline --help
For example, if you run ``bob bio baseline -vvv eigenface atnt``, it will run
the eigenface face recognition baseline on the atnt database (assuming you have
installed ```` and ``bob.db.atnt``).
To create your own baseline, you just need to define it like in the recipe
.. code-block:: py
from import Baseline
baseline = Baseline(name="my-baseline",
preprocessors={"default": 'my-preprocessor'},
Some databases may require some specific preprocessors depending on the type
of meta-informations provided. For instance, for some face recognition
databases, faces should be cropped in a particular way depending on the
annotations provided. To approach this issue, the preprocessors are defined in
a dictionary, with a generic preprocessor defined as **default** and the
database specific preprocessor defined by database name as in the example
.. code-block:: py
self.preprocessors = dict()
self.preprocessors["default"] = 'my-preprocessor'
self.preprocessors["database_name"] = 'my-specific-preprocessor'
Follow below a full example on how to define a baseline with database specific
.. code-block:: py
from import Baseline
preprocessors = {"default": 'my-preprocessor'}
preprocessors["database_name"] = 'my-specific-preprocessor'
baseline = Baseline(name="another-baseline",
.. note::
The triplet can be a resource or a configuration file. This works in the
same way as in :ref:`Running Experiments <running_part_1>`.
.. note::
Baselines are also registered as resources under the keyword
You can find the list of readily available baselines using the ````
.. code-block:: sh
$ --types baseline
......@@ -16,6 +16,7 @@ Base Classes
......@@ -80,4 +81,10 @@ Annotators
.. automodule::
.. automodule::
.. include:: links.rst
......@@ -75,6 +75,7 @@ Users Guide
......@@ -147,12 +147,19 @@ setup(
'dir =',
'gen =',
'evaluate =',
'baseline =',
# annotators
'': [
'dummy =',
'dummy =',
# Classifiers are important if you plan to distribute this package through
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment