Merge branch 'creating-baselines' into 'master'

Created the Baselines Concept

See merge request !151

bob/bio/base/__init__.py

@@ -6,6 +6,7 @@ from . import algorithm
 from . import tools
 from . import grid # only one file, not complete directory
 from . import annotator
+from . import baseline
 
 from . import script
 from . import test

bob/bio/base/baseline/Baseline.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+from .. import resource_keys, load_resource
+
+
+def search_preprocessor(db_name, keys):
+  """
+  Wrapper that searches for preprocessors for specific databases.
+  If not found, the default preprocessor is returned
+  """
+  for k in keys:
+    if db_name.startswith(k):
+      return k
+  else:
+    return "default"
+
+
+def get_available_databases():
+  """
+  Get all the available databases through the database entry-points
+  """
+
+  available_databases = dict()
+  all_databases = resource_keys('database', strip=[])
+  for database in all_databases:
+    try:
+      database_entry_point = load_resource(database, 'database')
+
+      available_databases[database] = dict()
+
+      # Checking if the database has data for the ZT normalization
+      available_databases[database]["has_zt"] = hasattr(database_entry_point, "zobjects") and hasattr(database_entry_point, "tobjects")
+      available_databases[database]["groups"] = []
+      # Searching for database groups
+      try:
+        groups = list(database_entry_point.groups())
+        for g in ["dev", "eval"]:
+          available_databases[database]["groups"] += [g] if g in groups else []
+      except Exception:
+        # In case the method groups is not implemented
+        available_databases[database]["groups"] = ["dev"]
+    except Exception:
+      pass
+  return available_databases
+
+
+class Baseline(object):
+  """
+  Base class to define baselines
+
+  A Baseline is composed by the triplet
+  :any:`bob.bio.base.preprocessor.Preprocessor`,
+  :any:`bob.bio.base.extractor.Extractor`, and
+  :any:`bob.bio.base.algorithm.Algorithm`
+
+  Attributes
+  ----------
+  name : str
+    Name of the baseline. This name will be displayed in the command line
+    interface.
+  preprocessors : dict
+    Dictionary containing all possible preprocessors
+  extractor : str
+    Registered resource or a config file containing the feature extractor
+  algorithm : str
+     Registered resource or a config file containing the algorithm
+  """
+
+  def __init__(self, name, preprocessors, extractor, algorithm, **kwargs):
+    super(Baseline, self).__init__(**kwargs)
+    self.name = name
+    self.preprocessors = preprocessors
+    self.extractor = extractor
+    self.algorithm = algorithm

bob/bio/base/baseline/__init__.py

+from .Baseline import Baseline, search_preprocessor, get_available_databases
+
+
+def get_config():
+  """Returns a string containing the configuration information.
+  """
+  import bob.extension
+  return bob.extension.get_config(__name__)
+
+
+# gets sphinx autodoc done right - don't remove it
+def __appropriate__(*args):
+  """Says object was actually declared here, and not in the import module.
+  Fixing sphinx warnings of not being able to find classes, when path is
+  shortened. Parameters:
+
+    *args: An iterable of objects to modify
+
+  Resolves `Sphinx referencing issues
+  <https://github.com/sphinx-doc/sphinx/issues/3048>`
+  """
+
+  for obj in args:
+    obj.__module__ = __name__
+
+
+__appropriate__(
+    Baseline,
+)
+
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/bio/base/script/baseline.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+
+"""
+A script to run biometric recognition baselines
+"""
+
+
+from .. import load_resource
+import os
+from .verify import main as verify
+from ..baseline import get_available_databases, search_preprocessor
+from bob.extension.scripts.click_helper import verbosity_option
+import click
+
+
+@click.command(context_settings={'ignore_unknown_options': True,
+                                 'allow_extra_args': True})
+@click.argument('baseline', required=True)
+@click.argument('database', required=True)
+@verbosity_option()
+@click.pass_context
+def baseline(ctx, baseline, database):
+    """Run a biometric recognition baseline.
+
+    \b
+    Example:
+        $ bob bio baseline eigenface atnt -vvv
+
+    which will run the eigenface baseline (from bob.bio.face) on the atnt
+    database.
+
+    \b
+    Check out all baselines available by running:
+    `resource.py --types baseline`
+    and all available databases by running:
+    `resource.py --types database`
+
+    This script accepts parameters accepted by verify.py as well.
+    See `verify.py --help` for the extra options that you can pass.
+
+    Hint: pass `--grid demanding` to run the baseline on the SGE grid.
+
+    Hint: pass `--temp-directory <dir>` to set the directory for temporary files
+
+    Hint: pass `--result-directory <dir>` to set the directory for resulting score files
+ 
+    """
+    # Triggering training for each baseline/database
+    loaded_baseline = load_resource(
+        baseline, 'baseline', package_prefix="bob.bio.")
+
+    # this is the default sub-directory that is used
+    sub_directory = os.path.join(database, baseline)
+
+    # find the compatible preprocessor for this database
+    database_data = get_available_databases()[database]
+    db = search_preprocessor(database, loaded_baseline.preprocessors.keys())
+    preprocessor = loaded_baseline.preprocessors[db]
+
+    # call verify with all parameters
+    parameters = [
+        '-p', preprocessor,
+        '-e', loaded_baseline.extractor,
+        '-d', database,
+        '-a', loaded_baseline.algorithm,
+        '--sub-directory', sub_directory
+    ] + ['-v'] * ctx.meta['verbosity']
+
+    parameters += ['--groups'] + database_data["groups"]
+
+    verify(parameters + ctx.args)

bob/bio/base/script/resources.py

@@ -9,8 +9,8 @@ def resources(command_line_parameters = None):
   import argparse
   parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
   parser.add_argument("--types", '-t', nargs = '+',
-                      choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator'),
-                      default = ('d', 'p', 'e', 'a', 'g', 'c', 'an'),
+                      choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator', 'b', 'baseline'),
+                      default = ('d', 'p', 'e', 'a', 'g', 'c', 'an', 'b'),
                       help = "Select the resource types that should be listed.")
 
   parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources")
@@ -55,6 +55,10 @@ def resources(command_line_parameters = None):
     print ("\nList of registered annotators:")
     print (bob.bio.base.list_resources('annotator', **kwargs))
 
+  if 'b' in args.types or 'baseline' in args.types:
+    print ("\nList of registered baseline:")
+    print (bob.bio.base.list_resources('baseline', **kwargs))
+
   print()
 
 def databases(command_line_parameters = None):

bob/bio/base/test/dummy/baseline.py

+from bob.bio.base.baseline import Baseline
+import pkg_resources
+import os
+
+dummy_dir = pkg_resources.resource_filename('bob.bio.base', 'test/dummy')
+baseline = Baseline(name="dummy", 
+                         preprocessors={"default": os.path.join(dummy_dir, 'preprocessor.py')},
+                         extractor=os.path.join(dummy_dir, 'extractor.py'),
+                         algorithm=os.path.join(dummy_dir, 'algorithm.py'))

bob/bio/base/test/test_baselines.py

+import tempfile
+import shutil
+from click.testing import CliRunner
+from bob.bio.base.script.baseline import baseline
+
+def test_baselines():
+
+    try:
+        tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
+        runner = CliRunner()
+        result = runner.invoke(baseline, args=('dummy', 'dummy', '-T', tmp_dir, '-R', tmp_dir))
+        assertion_error_message = (
+              'Command exited with this output: `{}\' \n'
+              'If the output is empty, you can run this script locally to see '
+              'what is wrong:\n'
+              'bin/bob bio baseline  -d dummy -a dummy -o /tmp/temp_annotations'
+              ''.format(result.output))
+        assert result.exit_code == 0, assertion_error_message
+
+    finally:
+        shutil.rmtree(tmp_dir)

bob/bio/base/utils/resources.py

@@ -21,7 +21,7 @@ logger = logging.getLogger("bob.bio.base")
 
 
 #: Keywords for which resources are defined.
-valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator')
+valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config', 'annotator', 'baseline')
 
 
 def _collect_config(paths):

doc/baseline.rst

+.. _bob.bio.base.baseline:
+
+==================
+Defining baselines
+==================
+
+
+Once you have a biometric system well established, tuned and working for a
+particular database (or a particular set of databases), you may want to provide
+**an easier to reproduce** way to share it. For this purpose, we defined
+something called baseline.
+
+A baseline (:any:`bob.bio.base.baseline.Baseline`) is composed by the triplet
+of :any:`bob.bio.base.preprocessor.Preprocessor`,
+:any:`bob.bio.base.extractor.Extractor` and
+:any:`bob.bio.base.algorithm.Algorithm`.
+
+First, check it out the baselines ready to be triggered in your environment by
+doing:
+
+.. code-block:: sh
+
+    $ bob bio baseline --help
+
+For example, if you run ``bob bio baseline -vvv eigenface atnt``, it will run
+the eigenface face recognition baseline on the atnt database (assuming you have
+installed ``bob.bio.face`` and ``bob.db.atnt``).
+
+
+To create your own baseline, you just need to define it like in the recipe
+below:
+
+.. code-block:: py
+
+    from bob.bio.base.baseline import Baseline
+
+    baseline = Baseline(name="my-baseline",
+                        preprocessors={"default": 'my-preprocessor'},
+                        extractor='my-extractor'),
+                        algorithm='my-algorithm'))
+
+Some databases may require some specific preprocessors depending on the type
+of meta-informations provided. For instance, for some face recognition
+databases, faces should be cropped in a particular way depending on the
+annotations provided. To approach this issue, the preprocessors are defined in
+a dictionary, with a generic preprocessor defined as **default** and the
+database specific preprocessor defined by database name as in the example
+below:
+
+.. code-block:: py
+
+    self.preprocessors = dict()
+    self.preprocessors["default"] = 'my-preprocessor'
+    self.preprocessors["database_name"] = 'my-specific-preprocessor'
+
+
+Follow below a full example on how to define a baseline with database specific
+preprocessors.
+
+.. code-block:: py
+
+    from bob.bio.base.baseline import Baseline
+
+    preprocessors = {"default": 'my-preprocessor'}
+    preprocessors["database_name"] = 'my-specific-preprocessor'
+    baseline = Baseline(name="another-baseline",
+                        preprocessors=preprocessors,
+                        extractor='my-extractor'),
+                        algorithm='my-algorithm'))
+
+.. note::
+
+   The triplet can be a resource or a configuration file. This works in the
+   same way as in :ref:`Running Experiments <running_part_1>`.
+
+.. note::
+
+  Baselines are also registered as resources under the keyword
+  `bob.bio.baseline`.
+
+You can find the list of readily available baselines using the ``resources.py``
+command:
+
+.. code-block:: sh
+
+    $ resources.py --types baseline

doc/implemented.rst

@@ -16,6 +16,7 @@ Base Classes
    bob.bio.base.algorithm.Algorithm
    bob.bio.base.grid.Grid
    bob.bio.base.annotator.Annotator
+   bob.bio.base.baseline.Baseline
 
 
 Implementations
@@ -80,4 +81,10 @@ Annotators
 .. automodule:: bob.bio.base.annotator
 
 
+Baselines
+---------
+
+.. automodule:: bob.bio.base.baseline
+
+
 .. include:: links.rst

doc/index.rst

@@ -75,6 +75,7 @@ Users Guide
    struct_bio_rec_sys
    experiments
    implementation
+   baseline
    filelist-guide
    more
    annotations

setup.py

@@ -147,12 +147,19 @@ setup(
         'dir               = bob.bio.base.script.commands:dir',
         'gen               = bob.bio.base.script.gen:gen',
         'evaluate          = bob.bio.base.script.commands:evaluate',
+        'baseline          = bob.bio.base.script.baseline:baseline',
       ],
 
       # annotators
       'bob.bio.annotator': [
         'dummy             = bob.bio.base.test.dummy.annotator:annotator',
       ],
+
+      #baselines
+      'bob.bio.baseline':[
+        'dummy = bob.bio.base.test.dummy.baseline:baseline',
+      ],
+
    },
 
     # Classifiers are important if you plan to distribute this package through