Commit 8b4068a2 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Solving other discussions

Made the baseline command more simple and moved code around

Improved documentation
parent 82413510
Pipeline #20222 passed with stage
in 21 minutes and 51 seconds
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
from .. import resource_keys, load_resource
def search_preprocessor(db_name, keys):
"""
Wrapper that searches for preprocessors for specific databases.
If not found, the default preprocessor is returned
"""
for k in keys:
if db_name.startswith(k):
return k
else:
return "default"
def get_available_databases():
"""
Get all the available databases through the database entry-points
"""
available_databases = dict()
all_databases = resource_keys('database', strip=[])
for database in all_databases:
try:
database_entry_point = load_resource(database, 'database')
available_databases[database] = dict()
# Checking if the database has data for the ZT normalization
available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
available_databases[database]["groups"] = []
# Searching for database groups
try:
groups = list(database_entry_point.groups())
for g in ["dev", "eval"]:
available_databases[database]["groups"] += [g] if g in groups else []
except Exception:
# In case the method groups is not implemented
available_databases[database]["groups"] = ["dev"]
except Exception:
pass
return available_databases
class Baseline(object):
"""
Base class to define baselines
A Baseline is composed by the triplet :any:`bob.bio.base.preprocessor.Preprocessor`,
:any:`bob.bio.base.extractor.Extractor` and :any:`bob.bio.base.algorithm.Algorithm`
Attributes
----------
name: str
Name of the baseline. This name will be displayed in the command line interface
preprocessors: dict
Dictionary containing all possible preprocessors
extractor: str
Registered resource or a config file containing the feature extractor
algorithm: str
Registered resource or a config file containing the algorithm
"""
def __init__(self, name="", preprocessors=dict(), extractor="", algorithm="", **kwargs):
super(Baseline, self).__init__(**kwargs)
self.name = name
self.preprocessors = preprocessors
self.extractor = extractor
self.algorithm = algorithm
"""
Base class to define baselines
A Baseline is composed by the triplet
:any:`bob.bio.base.preprocessor.Preprocessor`,
:any:`bob.bio.base.extractor.Extractor`, and
:any:`bob.bio.base.algorithm.Algorithm`
Attributes
----------
name : str
Name of the baseline. This name will be displayed in the command line
interface.
preprocessors : dict
Dictionary containing all possible preprocessors
extractor : str
Registered resource or a config file containing the feature extractor
algorithm : str
Registered resource or a config file containing the algorithm
"""
def __init__(self, name, preprocessors, extractor, algorithm, **kwargs):
super(Baseline, self).__init__(**kwargs)
self.name = name
self.preprocessors = preprocessors
self.extractor = extractor
self.algorithm = algorithm
from .Baseline import Baseline
import bob.bio.base
def get_available_databases():
"""
Get all the available databases through the database entry-points
"""
available_databases = dict()
all_databases = bob.bio.base.resource_keys('database', strip=[])
for database in all_databases:
try:
database_entry_point = bob.bio.base.load_resource(database, 'database')
available_databases[database] = dict()
# Checking if the database has data for the ZT normalization
available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
available_databases[database]["groups"] = []
# Searching for database groups
try:
groups = list(database_entry_point.groups())
for g in ["dev", "eval"]:
available_databases[database]["groups"] += [g] if g in groups else []
except:
# In case the method groups is not implemented
available_databases[database]["groups"] = ["dev"]
except:
pass
return available_databases
from .Baseline import Baseline, search_preprocessor, get_available_databases
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened. Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
__appropriate__(
Baseline,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -3,77 +3,71 @@
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
"""
This script runs some face recognition baselines under some face databases
Examples:
This command line will run the facenet from David Sandberg using the ATnT dataset:
`bob bio baseline --baseline facenet_msceleba_inception_v1 --database atnt`
A script to run biometric recognition baselines
"""
import bob.bio.base
import bob.io.base
from .. import load_resource
import os
from bob.bio.base.script.verify import main as verify
from bob.bio.base.baseline import get_available_databases
from bob.extension.scripts.click_helper import (
verbosity_option, ConfigCommand, ResourceOption)
from .verify import main as verify
from ..baseline import get_available_databases, search_preprocessor
from bob.extension.scripts.click_helper import verbosity_option
import click
@click.command(entry_point_group='bob.bio.config', cls=ConfigCommand)
@click.option('--database', '-d', required=True, cls=ResourceOption, help="Registered database. Check it out `resources.py --types database` for ready to be used databases")
@click.option('--baseline', '-b', required=True, cls=ResourceOption, help="Registered baseline. Check it out `resources.py --types baseline` for ready to be used baseline")
@click.option('--temp-dir', '-T', required=False, cls=ResourceOption, help="The directory for temporary files")
@click.option('--result-dir', '-R', required=False, cls=ResourceOption, help="The directory for resulting score files")
@click.option('--grid', '-g', help="Execute the algorithm in the SGE grid.", is_flag=True)
@click.option('--zt-norm', '-z', help="Enable the computation of ZT norms (if the database supports it).", is_flag=True)
@verbosity_option(cls=ResourceOption)
@click.command(context_settings={'ignore_unknown_options': True,
'allow_extra_args': True})
@click.argument('baseline', required=True)
@click.argument('database', required=True)
@verbosity_option()
@click.pass_context
def baseline(ctx, baseline, database):
"""Run a biometric recognition baseline.
def baseline(baseline, database, temp_dir, result_dir, grid, zt_norm, **kwargs):
"""
Run a biometric recognition baselines
\b
Example:
$ bob bio baseline eigenface atnt -vvv
Check it out all baselines available by typing `resource.py --types baseline`
which will run the eigenface baseline (from bob.bio.face) on the atnt
database.
"""
\b
Check out all baselines available by running:
`resource.py --types baseline`
and all available databases by running:
`resource.py --types database`
This script accepts parameters accepted by verify.py as well.
See `verify.py --help` for the extra options that you can pass.
Hint: pass `--grid demanding` to run the baseline on the SGE grid.
def search_preprocessor(key, keys):
"""
Wrapper that searches for preprocessors for specific databases.
If not found, the default preprocessor is returned
"""
for k in keys:
if key.startswith(k):
return k
else:
return "default"
Hint: pass `--temp-directory <dir>` to set the directory for temporary files
# Triggering training for each baseline/database
loaded_baseline = bob.bio.base.load_resource(baseline, 'baseline', package_prefix="bob.bio.")
Hint: pass `--result-directory <dir>` to set the directory for resulting score files
"""
# Triggering training for each baseline/database
loaded_baseline = load_resource(
baseline, 'baseline', package_prefix="bob.bio.")
# this is the default sub-directory that is used
sub_directory = os.path.join(database, baseline)
# find the compatible preprocessor for this database
database_data = get_available_databases()[database]
db = search_preprocessor(database, loaded_baseline.preprocessors.keys())
preprocessor = loaded_baseline.preprocessors[db]
# call verify with all parameters
parameters = [
'-p', loaded_baseline.preprocessors[search_preprocessor(database, loaded_baseline.preprocessors.keys())],
'-p', preprocessor,
'-e', loaded_baseline.extractor,
'-d', database,
'-a', loaded_baseline.algorithm,
'-vvv',
'--temp-directory', temp_dir,
'--result-directory', result_dir,
'--sub-directory', sub_directory
]
parameters += ['--groups'] + database_data["groups"]
if grid:
parameters += ['-g', 'demanding']
] + ['-v'] * ctx.meta['verbosity']
if zt_norm and 'has_zt' in database_data:
parameters += ['--zt-norm']
parameters += ['--groups'] + database_data["groups"]
verify(parameters)
verify(parameters + ctx.args)
......@@ -2,14 +2,8 @@ from bob.bio.base.baseline import Baseline
import pkg_resources
import os
dummy_dir = pkg_resources.resource_filename('bob.bio.base', 'test/dummy')
class DummyBaseline(Baseline):
def __init__(self, **kwargs):
super(DummyBaseline, self).__init__(**kwargs)
baseline = DummyBaseline(name="dummy",
baseline = Baseline(name="dummy",
preprocessors={"default": os.path.join(dummy_dir, 'preprocessor.py')},
extractor=os.path.join(dummy_dir, 'extractor.py'),
algorithm=os.path.join(dummy_dir, 'algorithm.py'))
......@@ -8,7 +8,7 @@ def test_baselines():
try:
tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
runner = CliRunner()
result = runner.invoke(baseline, args=('-d', 'dummy', '-b', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
result = runner.invoke(baseline, args=('dummy', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
assertion_error_message = (
'Command exited with this output: `{}\' \n'
'If the output is empty, you can run this script locally to see '
......@@ -16,6 +16,6 @@ def test_baselines():
'bin/bob bio baseline -d dummy -a dummy -o /tmp/temp_annotations'
''.format(result.output))
assert result.exit_code == 0, assertion_error_message
finally:
shutil.rmtree(tmp_dir)
......@@ -5,72 +5,81 @@ Defining baselines
==================
Once you have a biometric system well established, tuned and working for a particular database (or a particular set of databases), you may want to provide **an easier to reproduce** way to share it.
For this purpose, we defined something called baseline.
Once you have a biometric system well established, tuned and working for a
particular database (or a particular set of databases), you may want to provide
**an easier to reproduce** way to share it. For this purpose, we defined
something called baseline.
A baseline is composed by the triplet :any:`bob.bio.base.preprocessor.Preprocessor`, :any:`bob.bio.base.extractor.Extractor` and :any:`bob.bio.base.algorithm.Algorithm`.
A baseline (:any:`bob.bio.base.baseline.Baseline`) is composed by the triplet
of :any:`bob.bio.base.preprocessor.Preprocessor`,
:any:`bob.bio.base.extractor.Extractor` and
:any:`bob.bio.base.algorithm.Algorithm`.
First, check it out the baselines ready to be triggered in your environment by doing:
First, check it out the baselines ready to be triggered in your environment by
doing:
.. code-block:: sh
$ bob bio baseline --help
For example, if you run ``bob bio baseline -vvv eigenface atnt``, it will run
the eigenface face recognition baseline on the atnt database (assuming you have
installed ``bob.bio.face`` and ``bob.db.atnt``).
To create your own baseline, you just need to define it like in the recipe below:
To create your own baseline, you just need to define it like in the recipe
below:
.. code-block:: py
from bob.bio.base.baseline import Baseline
class DummyBaseline(Baseline):
def __init__(self):
self.preprocessors = dict() # SHOULD BE DEFINED AS A DICTIONARY
self.preprocessors["default"] = 'my-preprocessor'
self.extractor = 'my-extractor'
self.algorithm = 'my-algorithm'
baseline = DummyBaseline()
baseline = Baseline(name="my-baseline",
preprocessors={"default": 'my-preprocessor'},
extractor='my-extractor'),
algorithm='my-algorithm'))
Some databases may require some especific preprocessors depending on the type of meta-informations provided.
For instance, for some face recognition databases, faces should be cropped in a particular way depending on the annotations provided.
To approach this issue, the preprocessors are defined in a dictionary, with a generic preprocessor defined as **default** and the database specific preprocessor defined by database name as in the example below:
Some databases may require some specific preprocessors depending on the type
of meta-informations provided. For instance, for some face recognition
databases, faces should be cropped in a particular way depending on the
annotations provided. To approach this issue, the preprocessors are defined in
a dictionary, with a generic preprocessor defined as **default** and the
database specific preprocessor defined by database name as in the example
below:
.. code-block:: py
self.preprocessors = dict()
self.preprocessors["default"] = 'my-preprocessor'
self.preprocessors["database_name"] = 'my-specific-preprocessor'
Follow below a full example on how to define a baseline with database specific preprocessors.
Follow below a full example on how to define a baseline with database specific
preprocessors.
.. code-block:: py
from bob.bio.base.baseline import Baseline
class AnotherBaseline(Baseline):
def __init__(self):
self.preprocessors = dict() # SHOULD BE DEFINED AS A DICTIONARY
self.preprocessors["default"] = 'my-preprocessor'
self.preprocessors["database_name"] = 'my-specific-preprocessor'
self.extractor = 'my-extractor'
self.algorithm = 'my-algorithm'
baseline = AnotherBaseline()
preprocessors = {"default": 'my-preprocessor'}
preprocessors["database_name"] = 'my-specific-preprocessor'
baseline = Baseline(name="another-baseline",
preprocessors=preprocessors,
extractor='my-extractor'),
algorithm='my-algorithm'))
.. note::
The triplet can be a resource or a configuration file.
This works in the same way as in :ref:`Running Experiments <running_part_1>`.
The triplet can be a resource or a configuration file. This works in the
same way as in :ref:`Running Experiments <running_part_1>`.
.. note::
Baselines are also registered as resources under the keyworkd `bob.bio.baseline`.
Baselines are also registered as resources under the keyword
`bob.bio.baseline`.
You can find the list of readily available baselines using the ``resources.py`` command:
You can find the list of readily available baselines using the ``resources.py``
command:
.. code-block:: sh
......
......@@ -16,6 +16,7 @@ Base Classes
bob.bio.base.algorithm.Algorithm
bob.bio.base.grid.Grid
bob.bio.base.annotator.Annotator
bob.bio.base.baseline.Baseline
Implementations
......@@ -80,4 +81,10 @@ Annotators
.. automodule:: bob.bio.base.annotator
Baselines
---------
.. automodule:: bob.bio.base.baseline
.. include:: links.rst
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment