diff --git a/bob/bio/base/__init__.py b/bob/bio/base/__init__.py
index 9ef81253a184f70511b99f0c3ecda10d28de32a0..c69d5db39ea2a2281fe7a6e7b3994a17d42e0395 100644
--- a/bob/bio/base/__init__.py
+++ b/bob/bio/base/__init__.py
@@ -6,6 +6,7 @@ from . import algorithm
from . import tools
from . import grid # only one file, not complete directory
from . import annotator
+from . import baseline
from . import script
from . import test
diff --git a/bob/bio/base/baseline/Baseline.py b/bob/bio/base/baseline/Baseline.py
new file mode 100644
index 0000000000000000000000000000000000000000..40459ccad63c444709e6092b37d20b29e9048d33
--- /dev/null
+++ b/bob/bio/base/baseline/Baseline.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+from .. import resource_keys, load_resource
+
+
+def search_preprocessor(db_name, keys):
+ """
+ Wrapper that searches for preprocessors for specific databases.
+ If not found, the default preprocessor is returned
+ """
+ for k in keys:
+ if db_name.startswith(k):
+ return k
+ else:
+ return "default"
+
+
+def get_available_databases():
+ """
+ Get all the available databases through the database entry-points
+ """
+
+ available_databases = dict()
+ all_databases = resource_keys('database', strip=[])
+ for database in all_databases:
+ try:
+ database_entry_point = load_resource(database, 'database')
+
+ available_databases[database] = dict()
+
+ # Checking if the database has data for the ZT normalization
+ available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
+ available_databases[database]["groups"] = []
+ # Searching for database groups
+ try:
+ groups = list(database_entry_point.groups())
+ for g in ["dev", "eval"]:
+ available_databases[database]["groups"] += [g] if g in groups else []
+ except Exception:
+ # In case the method groups is not implemented
+ available_databases[database]["groups"] = ["dev"]
+ except Exception:
+ pass
+ return available_databases
+
+
+class Baseline(object):
+ """
+ Base class to define baselines
+
+ A Baseline is composed by the triplet
+ :any:`bob.bio.base.preprocessor.Preprocessor`,
+ :any:`bob.bio.base.extractor.Extractor`, and
+ :any:`bob.bio.base.algorithm.Algorithm`
+
+ Attributes
+ ----------
+ name : str
+ Name of the baseline. This name will be displayed in the command line
+ interface.
+ preprocessors : dict
+ Dictionary containing all possible preprocessors
+ extractor : str
+ Registered resource or a config file containing the feature extractor
+ algorithm : str
+ Registered resource or a config file containing the algorithm
+ """
+
+ def __init__(self, name, preprocessors, extractor, algorithm, **kwargs):
+ super(Baseline, self).__init__(**kwargs)
+ self.name = name
+ self.preprocessors = preprocessors
+ self.extractor = extractor
+ self.algorithm = algorithm
diff --git a/bob/bio/base/baseline/__init__.py b/bob/bio/base/baseline/__init__.py
new file mode 100755
index 0000000000000000000000000000000000000000..8e510bd3ea4dee434a12a3817a1c4769862d1fcd
--- /dev/null
+++ b/bob/bio/base/baseline/__init__.py
@@ -0,0 +1,31 @@
+from .Baseline import Baseline, search_preprocessor, get_available_databases
+
+
+def get_config():
+ """Returns a string containing the configuration information.
+ """
+ import bob.extension
+ return bob.extension.get_config(__name__)
+
+
+# gets sphinx autodoc done right - don't remove it
+def __appropriate__(*args):
+ """Says object was actually declared here, and not in the import module.
+ Fixing sphinx warnings of not being able to find classes, when path is
+ shortened. Parameters:
+
+ *args: An iterable of objects to modify
+
+ Resolves `Sphinx referencing issues
+ <https://github.com/sphinx-doc/sphinx/issues/3048>`
+ """
+
+ for obj in args:
+ obj.__module__ = __name__
+
+
+__appropriate__(
+ Baseline,
+)
+
+__all__ = [_ for _ in dir() if not _.startswith('_')]
diff --git a/bob/bio/base/script/baseline.py b/bob/bio/base/script/baseline.py
new file mode 100644
index 0000000000000000000000000000000000000000..74c219446b86748e2f5b0ce343f46c1332236e3b
--- /dev/null
+++ b/bob/bio/base/script/baseline.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+
+"""
+A script to run biometric recognition baselines
+"""
+
+
+from .. import load_resource
+import os
+from .verify import main as verify
+from ..baseline import get_available_databases, search_preprocessor
+from bob.extension.scripts.click_helper import verbosity_option
+import click
+
+
+@click.command(context_settings={'ignore_unknown_options': True,
+ 'allow_extra_args': True})
+@click.argument('baseline', required=True)
+@click.argument('database', required=True)
+@verbosity_option()
+@click.pass_context
+def baseline(ctx, baseline, database):
+ """Run a biometric recognition baseline.
+
+ \b
+ Example:
+ $ bob bio baseline eigenface atnt -vvv
+
+ which will run the eigenface baseline (from bob.bio.face) on the atnt
+ database.
+
+ \b
+ Check out all baselines available by running:
+ `resource.py --types baseline`
+ and all available databases by running:
+ `resource.py --types database`
+
+ This script accepts parameters accepted by verify.py as well.
+ See `verify.py --help` for the extra options that you can pass.
+
+ Hint: pass `--grid demanding` to run the baseline on the SGE grid.
+
+ Hint: pass `--temp-directory <dir>` to set the directory for temporary files
+
+ Hint: pass `--result-directory <dir>` to set the directory for resulting score files
+
+ """
+ # Triggering training for each baseline/database
+ loaded_baseline = load_resource(
+ baseline, 'baseline', package_prefix="bob.bio.")
+
+ # this is the default sub-directory that is used
+ sub_directory = os.path.join(database, baseline)
+
+ # find the compatible preprocessor for this database
+ database_data = get_available_databases()[database]
+ db = search_preprocessor(database, loaded_baseline.preprocessors.keys())
+ preprocessor = loaded_baseline.preprocessors[db]
+
+ # call verify with all parameters
+ parameters = [
+ '-p', preprocessor,
+ '-e', loaded_baseline.extractor,
+ '-d', database,
+ '-a', loaded_baseline.algorithm,
+ '--sub-directory', sub_directory
+ ] + ['-v'] * ctx.meta['verbosity']
+
+ parameters += ['--groups'] + database_data["groups"]
+
+ verify(parameters + ctx.args)
diff --git a/bob/bio/base/script/resources.py b/bob/bio/base/script/resources.py
index c68d6f22202a1f14ece2ecd572ff2dffed071289..f8f564511382ae7360ab1415900be84d71564820 100644
--- a/bob/bio/base/script/resources.py
+++ b/bob/bio/base/script/resources.py
@@ -9,8 +9,8 @@ def resources(command_line_parameters = None):
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--types", '-t', nargs = '+',
- choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator'),
- default = ('d', 'p', 'e', 'a', 'g', 'c', 'an'),
+ choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator', 'b', 'baseline'),
+ default = ('d', 'p', 'e', 'a', 'g', 'c', 'an', 'b'),
help = "Select the resource types that should be listed.")
parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources")
@@ -55,6 +55,10 @@ def resources(command_line_parameters = None):
print ("\nList of registered annotators:")
print (bob.bio.base.list_resources('annotator', **kwargs))
+ if 'b' in args.types or 'baseline' in args.types:
+ print ("\nList of registered baseline:")
+ print (bob.bio.base.list_resources('baseline', **kwargs))
+
print()
def databases(command_line_parameters = None):
diff --git a/bob/bio/base/test/dummy/baseline.py b/bob/bio/base/test/dummy/baseline.py
new file mode 100644
index 0000000000000000000000000000000000000000..e52d4b59992c160aa3a3a2b0c1874709968637a4
--- /dev/null
+++ b/bob/bio/base/test/dummy/baseline.py
@@ -0,0 +1,9 @@
+from bob.bio.base.baseline import Baseline
+import pkg_resources
+import os
+
+dummy_dir = pkg_resources.resource_filename('bob.bio.base', 'test/dummy')
+baseline = Baseline(name="dummy",
+ preprocessors={"default": os.path.join(dummy_dir, 'preprocessor.py')},
+ extractor=os.path.join(dummy_dir, 'extractor.py'),
+ algorithm=os.path.join(dummy_dir, 'algorithm.py'))
diff --git a/bob/bio/base/test/test_baselines.py b/bob/bio/base/test/test_baselines.py
new file mode 100644
index 0000000000000000000000000000000000000000..27dd5749b5eab5a7cbef8853753e25e41c218731
--- /dev/null
+++ b/bob/bio/base/test/test_baselines.py
@@ -0,0 +1,21 @@
+import tempfile
+import shutil
+from click.testing import CliRunner
+from bob.bio.base.script.baseline import baseline
+
+def test_baselines():
+
+ try:
+ tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
+ runner = CliRunner()
+ result = runner.invoke(baseline, args=('dummy', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
+ assertion_error_message = (
+ 'Command exited with this output: `{}\' \n'
+ 'If the output is empty, you can run this script locally to see '
+ 'what is wrong:\n'
+ 'bin/bob bio baseline -d dummy -a dummy -o /tmp/temp_annotations'
+ ''.format(result.output))
+ assert result.exit_code == 0, assertion_error_message
+
+ finally:
+ shutil.rmtree(tmp_dir)
diff --git a/bob/bio/base/utils/resources.py b/bob/bio/base/utils/resources.py
index fa5e3c42949e026c383fd475406f0471ad2d2770..57a5b5a294f5f67530c67b090e350df3cbfc0c2d 100644
--- a/bob/bio/base/utils/resources.py
+++ b/bob/bio/base/utils/resources.py
@@ -21,7 +21,7 @@ logger = logging.getLogger("bob.bio.base")
#: Keywords for which resources are defined.
-valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator')
+valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator', 'baseline')
def _collect_config(paths):
diff --git a/doc/baseline.rst b/doc/baseline.rst
new file mode 100644
index 0000000000000000000000000000000000000000..2e6da391465c2c217906ea450c3a1d415a407b6e
--- /dev/null
+++ b/doc/baseline.rst
@@ -0,0 +1,86 @@
+.. _bob.bio.base.baseline:
+
+==================
+Defining baselines
+==================
+
+
+Once you have a biometric system well established, tuned and working for a
+particular database (or a particular set of databases), you may want to provide
+**an easier to reproduce** way to share it. For this purpose, we defined
+something called baseline.
+
+A baseline (:any:`bob.bio.base.baseline.Baseline`) is composed by the triplet
+of :any:`bob.bio.base.preprocessor.Preprocessor`,
+:any:`bob.bio.base.extractor.Extractor` and
+:any:`bob.bio.base.algorithm.Algorithm`.
+
+First, check it out the baselines ready to be triggered in your environment by
+doing:
+
+.. code-block:: sh
+
+ $ bob bio baseline --help
+
+For example, if you run ``bob bio baseline -vvv eigenface atnt``, it will run
+the eigenface face recognition baseline on the atnt database (assuming you have
+installed ``bob.bio.face`` and ``bob.db.atnt``).
+
+
+To create your own baseline, you just need to define it like in the recipe
+below:
+
+.. code-block:: py
+
+ from bob.bio.base.baseline import Baseline
+
+ baseline = Baseline(name="my-baseline",
+ preprocessors={"default": 'my-preprocessor'},
+ extractor='my-extractor'),
+ algorithm='my-algorithm'))
+
+Some databases may require some specific preprocessors depending on the type
+of meta-informations provided. For instance, for some face recognition
+databases, faces should be cropped in a particular way depending on the
+annotations provided. To approach this issue, the preprocessors are defined in
+a dictionary, with a generic preprocessor defined as **default** and the
+database specific preprocessor defined by database name as in the example
+below:
+
+.. code-block:: py
+
+ self.preprocessors = dict()
+ self.preprocessors["default"] = 'my-preprocessor'
+ self.preprocessors["database_name"] = 'my-specific-preprocessor'
+
+
+Follow below a full example on how to define a baseline with database specific
+preprocessors.
+
+.. code-block:: py
+
+ from bob.bio.base.baseline import Baseline
+
+ preprocessors = {"default": 'my-preprocessor'}
+ preprocessors["database_name"] = 'my-specific-preprocessor'
+ baseline = Baseline(name="another-baseline",
+ preprocessors=preprocessors,
+ extractor='my-extractor'),
+ algorithm='my-algorithm'))
+
+.. note::
+
+ The triplet can be a resource or a configuration file. This works in the
+ same way as in :ref:`Running Experiments <running_part_1>`.
+
+.. note::
+
+ Baselines are also registered as resources under the keyword
+ `bob.bio.baseline`.
+
+You can find the list of readily available baselines using the ``resources.py``
+command:
+
+.. code-block:: sh
+
+ $ resources.py --types baseline
diff --git a/doc/implemented.rst b/doc/implemented.rst
index 052e56deb9b1f7c0bb0f8fb12aafc59215c1cb82..80a4ab1ee8a35d73c98bafae3c2261ac66184d01 100644
--- a/doc/implemented.rst
+++ b/doc/implemented.rst
@@ -16,6 +16,7 @@ Base Classes
bob.bio.base.algorithm.Algorithm
bob.bio.base.grid.Grid
bob.bio.base.annotator.Annotator
+ bob.bio.base.baseline.Baseline
Implementations
@@ -80,4 +81,10 @@ Annotators
.. automodule:: bob.bio.base.annotator
+Baselines
+---------
+
+.. automodule:: bob.bio.base.baseline
+
+
.. include:: links.rst
diff --git a/doc/index.rst b/doc/index.rst
index 2b966a5d00794c070ff4578ccbe667253e593a4f..8e1e3a67bfc8117c13f1073338e3f209f2dc16f2 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -75,6 +75,7 @@ Users Guide
struct_bio_rec_sys
experiments
implementation
+ baseline
filelist-guide
more
annotations
diff --git a/setup.py b/setup.py
index 58dbbfcf7880f7182c00812da011b0704b980d21..e488a1c3698b2554a2a5ff1d6ff8b2e35adce0d0 100644
--- a/setup.py
+++ b/setup.py
@@ -147,12 +147,19 @@ setup(
'dir = bob.bio.base.script.commands:dir',
'gen = bob.bio.base.script.gen:gen',
'evaluate = bob.bio.base.script.commands:evaluate',
+ 'baseline = bob.bio.base.script.baseline:baseline',
],
# annotators
'bob.bio.annotator': [
'dummy = bob.bio.base.test.dummy.annotator:annotator',
],
+
+ #baselines
+ 'bob.bio.baseline':[
+ 'dummy = bob.bio.base.test.dummy.baseline:baseline',
+ ],
+
},
# Classifiers are important if you plan to distribute this package through