Implement set of commands for bio.base based on bob.measure

bob/bio/base/score/load.py

@@ -354,7 +354,6 @@ def cmc(filename, ncolumns=None):
     assert ncolumns == 5
     return cmc_five_column(filename)
 
-
 def load_score(filename, ncolumns=None, minimal=False, **kwargs):
   """Load scores using numpy.loadtxt and return the data as a numpy array.
 
@@ -419,6 +418,37 @@ def load_score(filename, ncolumns=None, minimal=False, **kwargs):
   score_lines = numpy.array(score_lines, new_dtype)
   return score_lines
 
+def load_files(filenames, func_load):
+    """Load a list of score files and return a list of tuples of (neg, pos)
+
+    Parameters
+    ----------
+
+    filenames : :any:`list`
+        list of file paths
+    func_load :
+        function that can read files in the list
+
+    Returns
+    -------
+
+    :any:`list`: [(neg,pos)] A list of tuples, where each tuple contains the
+    ``negative`` and ``positive`` sceach system/probee.
+
+    """
+    if filenames is None:
+        return None
+    res = []
+    for filepath in filenames:
+        try:
+            tmp = func_load(filepath)
+            if isinstance(tmp, list):
+                res += func_load(filepath)
+            else:
+                res.append(tmp)
+        except:
+            raise
+    return res
 
 def get_negatives_positives(score_lines):
   """Take the output of load_score and return negatives and positives.  This

bob/bio/base/script/commands.py

+''' Click commands for ``bob.bio.base`` '''
+
+
+import click
+from ..score import load
+from bob.measure.script import figure
+from bob.measure.script import common_options
+from bob.extension.scripts.click_helper import verbosity_option
+
+
+FUNC_SPLIT = lambda x: load.load_files(x, load.split)
+
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.table_option()
+@common_options.test_option()
+@common_options.open_file_mode_option()
+@common_options.output_plot_metric_option()
+@common_options.criterion_option()
+@common_options.threshold_option()
+@verbosity_option()
+@click.pass_context
+def metrics(ctx, scores, test, **kargs):
+    """Prints a single output line that contains all info for a given
+    criterion (eer or hter).
+
+    You need provide one or more development score file(s) for each experiment.
+    You can also provide test files along with dev files but the flag `--test`
+    is required in that case. Files must be 4- or 5- columns format, see
+    :py:func:`bob.bio.base.score.load.four_column` and
+    :py:func:`bob.bio.base.score.load.five_column` for details.
+
+    Resulting table format can be changer using the `--tablefmt`. Default
+    formats are `rst` when output in the terminal and `latex` when
+    written in a log file (see `--log`)
+
+    Examples:
+        $ bob bio metrics dev-scores
+
+        $ bob bio metrics --test -l results.txt dev-scores1 test-scores1
+
+        $ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
+    """
+    process = figure.Metrics(ctx, scores, test, FUNC_SPLIT)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.titles_option()
+@common_options.sep_dev_test_option()
+@common_options.output_plot_file_option(default_out='roc.pdf')
+@common_options.test_option()
+@common_options.points_curve_option()
+@common_options.semilogx_option(True)
+@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
+@common_options.axis_fontsize_option()
+@common_options.x_rotation_option()
+@common_options.fmr_line_at_option()
+@verbosity_option()
+@click.pass_context
+def roc(ctx, scores, test, **kargs):
+    """Plot ROC (receiver operating characteristic) curve:
+    The plot will represent the false match rate on the horizontal axis and the
+    false non match rate on the vertical axis.  The values for the axis will be
+    computed using :py:func:`bob.measure.roc`.
+
+    You need provide one or more development score file(s) for each experiment.
+    You can also provide test files along with dev files but the flag `--test`
+    is required in that case.Files must be 4- or 5- columns format, see
+    :py:func:`bob.bio.base.score.load.four_column` and
+    :py:func:`bob.bio.base.score.load.five_column` for details.
+
+
+    Examples:
+        $ bob bio roc dev-scores
+
+        $ bob bio roc --test dev-scores1 test-scores1 dev-scores2
+        test-scores2
+
+        $ bob bio roc --test -o my_roc.pdf dev-scores1 test-scores1
+    """
+    process = figure.Roc(ctx, scores, test, FUNC_SPLIT)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='det.pdf')
+@common_options.titles_option()
+@common_options.sep_dev_test_option()
+@common_options.test_option()
+@common_options.axis_fontsize_option(dflt=6)
+@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
+@common_options.x_rotation_option(dflt=45)
+@common_options.points_curve_option()
+@verbosity_option()
+@click.pass_context
+def det(ctx, scores, test, **kargs):
+    """Plot DET (detection error trade-off) curve:
+    modified ROC curve which plots error rates on both axes
+    (false positives on the x-axis and false negatives on the y-axis)
+
+    You need provide one or more development score file(s) for each experiment.
+    You can also provide test files along with dev files but the flag `--test`
+    is required in that case. Files must be 4- or 5- columns format, see
+    :py:func:`bob.bio.base.score.load.four_column` and
+    :py:func:`bob.bio.base.score.load.five_column` for details.
+
+
+    Examples:
+        $ bob bio det dev-scores
+
+        $ bob bio det --test dev-scores1 test-scores1 dev-scores2
+        test-scores2
+
+        $ bob bio det --test -o my_det.pdf dev-scores1 test-scores1
+    """
+    process = figure.Det(ctx, scores, test, FUNC_SPLIT)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(test_mandatory=True, nargs=-1)
+@common_options.output_plot_file_option(default_out='epc.pdf')
+@common_options.titles_option()
+@common_options.points_curve_option()
+@common_options.axis_fontsize_option()
+@verbosity_option()
+@click.pass_context
+def epc(ctx, scores, **kargs):
+    """Plot EPC (expected performance curve):
+    plots the error rate on the test set depending on a threshold selected
+    a-priori on the development set and accounts for varying relative cost
+    in [0; 1] of FPR and FNR when calculating the threshold.
+
+    You need provide one or more development score and test file(s)
+    for each experiment. Files must be 4- or 5- columns format, see
+    :py:func:`bob.bio.base.score.load.four_column` and
+    :py:func:`bob.bio.base.score.load.five_column` for details.
+
+    Examples:
+        $ bob bio epc dev-scores test-scores
+
+        $ bob bio epc -o my_epc.pdf dev-scores1 test-scores1
+    """
+    process = figure.Epc(ctx, scores, True, FUNC_SPLIT)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='hist.pdf')
+@common_options.test_option()
+@common_options.n_bins_option()
+@common_options.criterion_option()
+@common_options.axis_fontsize_option()
+@common_options.threshold_option()
+@verbosity_option()
+@click.pass_context
+def hist(ctx, scores, test, **kwargs):
+    """ Plots histograms of positive and negatives along with threshold
+    criterion.
+
+    You need provide one or more development score file(s) for each experiment.
+    You can also provide test files along with dev files but the flag `--test`
+    is required in that case.
+
+    Examples:
+        $ bob bio hist dev-scores
+
+        $ bob bio hist --test dev-scores1 test-scores1 dev-scores2
+        test-scores2
+
+        $ bob bio hist --test --criter hter dev-scores1 test-scores1
+    """
+    process = figure.Hist(ctx, scores, test, FUNC_SPLIT)
+    process.run()

bob/bio/base/test/test_commands.py

+'''Tests for bob.measure scripts'''
+
+import sys
+import filecmp
+import click
+from click.testing import CliRunner
+import pkg_resources
+from ..script import commands
+
+def test_metrics():
+    dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.txt')
+    runner = CliRunner()
+    result = runner.invoke(commands.metrics, [dev1])
+    with runner.isolated_filesystem():
+        with open('tmp', 'w') as f:
+            f.write(result.output)
+        assert result.exit_code == 0
+    dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-5col.txt')
+    test1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-4col.txt')
+    test2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-5col.txt')
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['--test', dev1, test1, dev2, test2]
+        )
+        with open('tmp', 'w') as f:
+            f.write(result.output)
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
+        )
+        assert result.exit_code == 0
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
+        )
+        assert result.exit_code == 0
+
+def test_roc():
+    dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.txt')
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.roc, ['--output','test.pdf',dev1])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+    dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-5col.txt')
+    test1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-4col.txt')
+    test2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-5col.txt')
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.roc, ['--test', '--split', '--output',
+                                              'test.pdf',
+                                              dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.roc, ['--test', '--output',
+                                              'test.pdf', '--titles', 'A,B', 
+                                              dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+
+def test_det():
+    dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.txt')
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.det, [dev1])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+    dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-5col.txt')
+    test1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-4col.txt')
+    test2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-5col.txt')
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.det, ['--test', '--split', '--output',
+                                              'test.pdf', '--titles', 'A,B',
+                                              dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.det, ['--test', '--output',
+                                              'test.pdf',
+                                              dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+def test_epc():
+    dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.txt')
+    test1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-4col.txt')
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.epc, [dev1, test1])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+    dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.tar.gz')
+    test2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-5col.txt')
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.epc, ['--output', 'test.pdf',
+                                              '--titles', 'A,B',
+                                              dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+def test_hist():
+    dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-4col.txt')
+    dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                           'data/dev-5col.txt')
+    test1 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-4col.txt')
+    test2 = pkg_resources.resource_filename('bob.bio.base.test',
+                                            'data/test-5col.txt')
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.hist, [dev1])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
+                                               'HISTO.pdf', '-b', 30,
+                                               dev1, dev2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
+                                               'HISTO.pdf', '-b', 30, '-F',
+                                               3, dev1, test1, dev2, test2])
+        if result.output:
+            click.echo(result.output)
+        assert result.exit_code == 0

setup.py

@@ -73,7 +73,6 @@ setup(
         'verify.py         = bob.bio.base.script.verify:main',
         'resources.py      = bob.bio.base.script.resources:resources',
         'databases.py      = bob.bio.base.script.resources:databases',
-        'evaluate.py       = bob.bio.base.script.evaluate:main',
         'collect_results.py = bob.bio.base.script.collect_results:main',
         'grid_search.py    = bob.bio.base.script.grid_search:main',
         'preprocess.py     = bob.bio.base.script.preprocess:main',
@@ -139,6 +138,12 @@ setup(
       # bob bio scripts
       'bob.bio.cli': [
         'annotate          = bob.bio.base.script.annotate:annotate',
+        'evaluate          = bob.bio.base.script.evaluate:evaluate',
+        'metrics           = bob.bio.base.script.commands:metrics',
+        'roc               = bob.bio.base.script.commands:roc',
+        'det               = bob.bio.base.script.commands:det',
+        'epc               = bob.bio.base.script.commands:epc',
+        'hist               = bob.bio.base.script.commands:hist',
       ],
 
       # annotators