Commit 79d8bc26 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'creating-baselines' into 'master'

Created the Baselines Concept

See merge request !151
parents 8f48d57f 8b4068a2
Pipeline #20229 canceled with stage
......@@ -6,6 +6,7 @@ from . import algorithm
from . import tools
from . import grid # only one file, not complete directory
from . import annotator
from . import baseline
from . import script
from . import test
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
from .. import resource_keys, load_resource
def search_preprocessor(db_name, keys):
"""
Wrapper that searches for preprocessors for specific databases.
If not found, the default preprocessor is returned
"""
for k in keys:
if db_name.startswith(k):
return k
else:
return "default"
def get_available_databases():
"""
Get all the available databases through the database entry-points
"""
available_databases = dict()
all_databases = resource_keys('database', strip=[])
for database in all_databases:
try:
database_entry_point = load_resource(database, 'database')
available_databases[database] = dict()
# Checking if the database has data for the ZT normalization
available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
available_databases[database]["groups"] = []
# Searching for database groups
try:
groups = list(database_entry_point.groups())
for g in ["dev", "eval"]:
available_databases[database]["groups"] += [g] if g in groups else []
except Exception:
# In case the method groups is not implemented
available_databases[database]["groups"] = ["dev"]
except Exception:
pass
return available_databases
class Baseline(object):
"""
Base class to define baselines
A Baseline is composed by the triplet
:any:`bob.bio.base.preprocessor.Preprocessor`,
:any:`bob.bio.base.extractor.Extractor`, and
:any:`bob.bio.base.algorithm.Algorithm`
Attributes
----------
name : str
Name of the baseline. This name will be displayed in the command line
interface.
preprocessors : dict
Dictionary containing all possible preprocessors
extractor : str
Registered resource or a config file containing the feature extractor
algorithm : str
Registered resource or a config file containing the algorithm
"""
def __init__(self, name, preprocessors, extractor, algorithm, **kwargs):
super(Baseline, self).__init__(**kwargs)
self.name = name
self.preprocessors = preprocessors
self.extractor = extractor
self.algorithm = algorithm
from .Baseline import Baseline, search_preprocessor, get_available_databases
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened. Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
__appropriate__(
Baseline,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
"""
A script to run biometric recognition baselines
"""
from .. import load_resource
import os
from .verify import main as verify
from ..baseline import get_available_databases, search_preprocessor
from bob.extension.scripts.click_helper import verbosity_option
import click
@click.command(context_settings={'ignore_unknown_options': True,
'allow_extra_args': True})
@click.argument('baseline', required=True)
@click.argument('database', required=True)
@verbosity_option()
@click.pass_context
def baseline(ctx, baseline, database):
"""Run a biometric recognition baseline.
\b
Example:
$ bob bio baseline eigenface atnt -vvv
which will run the eigenface baseline (from bob.bio.face) on the atnt
database.
\b
Check out all baselines available by running:
`resource.py --types baseline`
and all available databases by running:
`resource.py --types database`
This script accepts parameters accepted by verify.py as well.
See `verify.py --help` for the extra options that you can pass.
Hint: pass `--grid demanding` to run the baseline on the SGE grid.
Hint: pass `--temp-directory <dir>` to set the directory for temporary files
Hint: pass `--result-directory <dir>` to set the directory for resulting score files
"""
# Triggering training for each baseline/database
loaded_baseline = load_resource(
baseline, 'baseline', package_prefix="bob.bio.")
# this is the default sub-directory that is used
sub_directory = os.path.join(database, baseline)
# find the compatible preprocessor for this database
database_data = get_available_databases()[database]
db = search_preprocessor(database, loaded_baseline.preprocessors.keys())
preprocessor = loaded_baseline.preprocessors[db]
# call verify with all parameters
parameters = [
'-p', preprocessor,
'-e', loaded_baseline.extractor,
'-d', database,
'-a', loaded_baseline.algorithm,
'--sub-directory', sub_directory
] + ['-v'] * ctx.meta['verbosity']
parameters += ['--groups'] + database_data["groups"]
verify(parameters + ctx.args)
......@@ -9,8 +9,8 @@ def resources(command_line_parameters = None):
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--types", '-t', nargs = '+',
choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator'),
default = ('d', 'p', 'e', 'a', 'g', 'c', 'an'),
choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator', 'b', 'baseline'),
default = ('d', 'p', 'e', 'a', 'g', 'c', 'an', 'b'),
help = "Select the resource types that should be listed.")
parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources")
......@@ -55,6 +55,10 @@ def resources(command_line_parameters = None):
print ("\nList of registered annotators:")
print (bob.bio.base.list_resources('annotator', **kwargs))
if 'b' in args.types or 'baseline' in args.types:
print ("\nList of registered baseline:")
print (bob.bio.base.list_resources('baseline', **kwargs))
print()
def databases(command_line_parameters = None):
......
from bob.bio.base.baseline import Baseline
import pkg_resources
import os
dummy_dir = pkg_resources.resource_filename('bob.bio.base', 'test/dummy')
baseline = Baseline(name="dummy",
preprocessors={"default": os.path.join(dummy_dir, 'preprocessor.py')},
extractor=os.path.join(dummy_dir, 'extractor.py'),
algorithm=os.path.join(dummy_dir, 'algorithm.py'))
import tempfile
import shutil
from click.testing import CliRunner
from bob.bio.base.script.baseline import baseline
def test_baselines():
try:
tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
runner = CliRunner()
result = runner.invoke(baseline, args=('dummy', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
assertion_error_message = (
'Command exited with this output: `{}\' \n'
'If the output is empty, you can run this script locally to see '
'what is wrong:\n'
'bin/bob bio baseline -d dummy -a dummy -o /tmp/temp_annotations'
''.format(result.output))
assert result.exit_code == 0, assertion_error_message
finally:
shutil.rmtree(tmp_dir)
......@@ -21,7 +21,7 @@ logger = logging.getLogger("bob.bio.base")
#: Keywords for which resources are defined.
valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator')
valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator', 'baseline')
def _collect_config(paths):
......
.. _bob.bio.base.baseline:
==================
Defining baselines
==================
Once you have a biometric system well established, tuned and working for a
particular database (or a particular set of databases), you may want to provide
**an easier to reproduce** way to share it. For this purpose, we defined
something called baseline.
A baseline (:any:`bob.bio.base.baseline.Baseline`) is composed by the triplet
of :any:`bob.bio.base.preprocessor.Preprocessor`,
:any:`bob.bio.base.extractor.Extractor` and
:any:`bob.bio.base.algorithm.Algorithm`.
First, check it out the baselines ready to be triggered in your environment by
doing:
.. code-block:: sh
$ bob bio baseline --help
For example, if you run ``bob bio baseline -vvv eigenface atnt``, it will run
the eigenface face recognition baseline on the atnt database (assuming you have
installed ``bob.bio.face`` and ``bob.db.atnt``).
To create your own baseline, you just need to define it like in the recipe
below:
.. code-block:: py
from bob.bio.base.baseline import Baseline
baseline = Baseline(name="my-baseline",
preprocessors={"default": 'my-preprocessor'},
extractor='my-extractor'),
algorithm='my-algorithm'))
Some databases may require some specific preprocessors depending on the type
of meta-informations provided. For instance, for some face recognition
databases, faces should be cropped in a particular way depending on the
annotations provided. To approach this issue, the preprocessors are defined in
a dictionary, with a generic preprocessor defined as **default** and the
database specific preprocessor defined by database name as in the example
below:
.. code-block:: py
self.preprocessors = dict()
self.preprocessors["default"] = 'my-preprocessor'
self.preprocessors["database_name"] = 'my-specific-preprocessor'
Follow below a full example on how to define a baseline with database specific
preprocessors.
.. code-block:: py
from bob.bio.base.baseline import Baseline
preprocessors = {"default": 'my-preprocessor'}
preprocessors["database_name"] = 'my-specific-preprocessor'
baseline = Baseline(name="another-baseline",
preprocessors=preprocessors,
extractor='my-extractor'),
algorithm='my-algorithm'))
.. note::
The triplet can be a resource or a configuration file. This works in the
same way as in :ref:`Running Experiments <running_part_1>`.
.. note::
Baselines are also registered as resources under the keyword
`bob.bio.baseline`.
You can find the list of readily available baselines using the ``resources.py``
command:
.. code-block:: sh
$ resources.py --types baseline
......@@ -16,6 +16,7 @@ Base Classes
bob.bio.base.algorithm.Algorithm
bob.bio.base.grid.Grid
bob.bio.base.annotator.Annotator
bob.bio.base.baseline.Baseline
Implementations
......@@ -80,4 +81,10 @@ Annotators
.. automodule:: bob.bio.base.annotator
Baselines
---------
.. automodule:: bob.bio.base.baseline
.. include:: links.rst
......@@ -75,6 +75,7 @@ Users Guide
struct_bio_rec_sys
experiments
implementation
baseline
filelist-guide
more
annotations
......
......@@ -147,12 +147,19 @@ setup(
'dir = bob.bio.base.script.commands:dir',
'gen = bob.bio.base.script.gen:gen',
'evaluate = bob.bio.base.script.commands:evaluate',
'baseline = bob.bio.base.script.baseline:baseline',
],
# annotators
'bob.bio.annotator': [
'dummy = bob.bio.base.test.dummy.annotator:annotator',
],
#baselines
'bob.bio.baseline':[
'dummy = bob.bio.base.test.dummy.baseline:baseline',
],
},
# Classifiers are important if you plan to distribute this package through
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment