From 3b18e032bd8d7b8ce9921642f4e1f0cb225dee70 Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Fri, 6 Apr 2018 08:15:37 +0200
Subject: [PATCH] Implement set of commands for bio.base based on bob.measure
---
bob/bio/base/score/load.py | 32 +++++-
bob/bio/base/script/commands.py | 175 +++++++++++++++++++++++++++++
bob/bio/base/test/test_commands.py | 159 ++++++++++++++++++++++++++
setup.py | 7 +-
4 files changed, 371 insertions(+), 2 deletions(-)
create mode 100644 bob/bio/base/script/commands.py
create mode 100644 bob/bio/base/test/test_commands.py
diff --git a/bob/bio/base/score/load.py b/bob/bio/base/score/load.py
index 7a5d76ca..6bc3987e 100644
--- a/bob/bio/base/score/load.py
+++ b/bob/bio/base/score/load.py
@@ -354,7 +354,6 @@ def cmc(filename, ncolumns=None):
assert ncolumns == 5
return cmc_five_column(filename)
-
def load_score(filename, ncolumns=None, minimal=False, **kwargs):
"""Load scores using numpy.loadtxt and return the data as a numpy array.
@@ -419,6 +418,37 @@ def load_score(filename, ncolumns=None, minimal=False, **kwargs):
score_lines = numpy.array(score_lines, new_dtype)
return score_lines
+def load_files(filenames, func_load):
+ """Load a list of score files and return a list of tuples of (neg, pos)
+
+ Parameters
+ ----------
+
+ filenames : :any:`list`
+ list of file paths
+ func_load :
+ function that can read files in the list
+
+ Returns
+ -------
+
+ :any:`list`: [(neg,pos)] A list of tuples, where each tuple contains the
+ ``negative`` and ``positive`` sceach system/probee.
+
+ """
+ if filenames is None:
+ return None
+ res = []
+ for filepath in filenames:
+ try:
+ tmp = func_load(filepath)
+ if isinstance(tmp, list):
+ res += func_load(filepath)
+ else:
+ res.append(tmp)
+ except:
+ raise
+ return res
def get_negatives_positives(score_lines):
"""Take the output of load_score and return negatives and positives. This
diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
new file mode 100644
index 00000000..c90683d7
--- /dev/null
+++ b/bob/bio/base/script/commands.py
@@ -0,0 +1,175 @@
+''' Click commands for ``bob.bio.base`` '''
+
+
+import click
+from ..score import load
+from bob.measure.script import figure
+from bob.measure.script import common_options
+from bob.extension.scripts.click_helper import verbosity_option
+
+
+FUNC_SPLIT = lambda x: load.load_files(x, load.split)
+
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.table_option()
+@common_options.test_option()
+@common_options.open_file_mode_option()
+@common_options.output_plot_metric_option()
+@common_options.criterion_option()
+@common_options.threshold_option()
+@verbosity_option()
+@click.pass_context
+def metrics(ctx, scores, test, **kargs):
+ """Prints a single output line that contains all info for a given
+ criterion (eer or hter).
+
+ You need provide one or more development score file(s) for each experiment.
+ You can also provide test files along with dev files but the flag `--test`
+ is required in that case. Files must be 4- or 5- columns format, see
+ :py:func:`bob.bio.base.score.load.four_column` and
+ :py:func:`bob.bio.base.score.load.five_column` for details.
+
+ Resulting table format can be changer using the `--tablefmt`. Default
+ formats are `rst` when output in the terminal and `latex` when
+ written in a log file (see `--log`)
+
+ Examples:
+ $ bob bio metrics dev-scores
+
+ $ bob bio metrics --test -l results.txt dev-scores1 test-scores1
+
+ $ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
+ """
+ process = figure.Metrics(ctx, scores, test, FUNC_SPLIT)
+ process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.titles_option()
+@common_options.sep_dev_test_option()
+@common_options.output_plot_file_option(default_out='roc.pdf')
+@common_options.test_option()
+@common_options.points_curve_option()
+@common_options.semilogx_option(True)
+@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
+@common_options.axis_fontsize_option()
+@common_options.x_rotation_option()
+@common_options.fmr_line_at_option()
+@verbosity_option()
+@click.pass_context
+def roc(ctx, scores, test, **kargs):
+ """Plot ROC (receiver operating characteristic) curve:
+ The plot will represent the false match rate on the horizontal axis and the
+ false non match rate on the vertical axis. The values for the axis will be
+ computed using :py:func:`bob.measure.roc`.
+
+ You need provide one or more development score file(s) for each experiment.
+ You can also provide test files along with dev files but the flag `--test`
+ is required in that case.Files must be 4- or 5- columns format, see
+ :py:func:`bob.bio.base.score.load.four_column` and
+ :py:func:`bob.bio.base.score.load.five_column` for details.
+
+
+ Examples:
+ $ bob bio roc dev-scores
+
+ $ bob bio roc --test dev-scores1 test-scores1 dev-scores2
+ test-scores2
+
+ $ bob bio roc --test -o my_roc.pdf dev-scores1 test-scores1
+ """
+ process = figure.Roc(ctx, scores, test, FUNC_SPLIT)
+ process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='det.pdf')
+@common_options.titles_option()
+@common_options.sep_dev_test_option()
+@common_options.test_option()
+@common_options.axis_fontsize_option(dflt=6)
+@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
+@common_options.x_rotation_option(dflt=45)
+@common_options.points_curve_option()
+@verbosity_option()
+@click.pass_context
+def det(ctx, scores, test, **kargs):
+ """Plot DET (detection error trade-off) curve:
+ modified ROC curve which plots error rates on both axes
+ (false positives on the x-axis and false negatives on the y-axis)
+
+ You need provide one or more development score file(s) for each experiment.
+ You can also provide test files along with dev files but the flag `--test`
+ is required in that case. Files must be 4- or 5- columns format, see
+ :py:func:`bob.bio.base.score.load.four_column` and
+ :py:func:`bob.bio.base.score.load.five_column` for details.
+
+
+ Examples:
+ $ bob bio det dev-scores
+
+ $ bob bio det --test dev-scores1 test-scores1 dev-scores2
+ test-scores2
+
+ $ bob bio det --test -o my_det.pdf dev-scores1 test-scores1
+ """
+ process = figure.Det(ctx, scores, test, FUNC_SPLIT)
+ process.run()
+
+@click.command()
+@common_options.scores_argument(test_mandatory=True, nargs=-1)
+@common_options.output_plot_file_option(default_out='epc.pdf')
+@common_options.titles_option()
+@common_options.points_curve_option()
+@common_options.axis_fontsize_option()
+@verbosity_option()
+@click.pass_context
+def epc(ctx, scores, **kargs):
+ """Plot EPC (expected performance curve):
+ plots the error rate on the test set depending on a threshold selected
+ a-priori on the development set and accounts for varying relative cost
+ in [0; 1] of FPR and FNR when calculating the threshold.
+
+ You need provide one or more development score and test file(s)
+ for each experiment. Files must be 4- or 5- columns format, see
+ :py:func:`bob.bio.base.score.load.four_column` and
+ :py:func:`bob.bio.base.score.load.five_column` for details.
+
+ Examples:
+ $ bob bio epc dev-scores test-scores
+
+ $ bob bio epc -o my_epc.pdf dev-scores1 test-scores1
+ """
+ process = figure.Epc(ctx, scores, True, FUNC_SPLIT)
+ process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='hist.pdf')
+@common_options.test_option()
+@common_options.n_bins_option()
+@common_options.criterion_option()
+@common_options.axis_fontsize_option()
+@common_options.threshold_option()
+@verbosity_option()
+@click.pass_context
+def hist(ctx, scores, test, **kwargs):
+ """ Plots histograms of positive and negatives along with threshold
+ criterion.
+
+ You need provide one or more development score file(s) for each experiment.
+ You can also provide test files along with dev files but the flag `--test`
+ is required in that case.
+
+ Examples:
+ $ bob bio hist dev-scores
+
+ $ bob bio hist --test dev-scores1 test-scores1 dev-scores2
+ test-scores2
+
+ $ bob bio hist --test --criter hter dev-scores1 test-scores1
+ """
+ process = figure.Hist(ctx, scores, test, FUNC_SPLIT)
+ process.run()
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
new file mode 100644
index 00000000..4ef3d018
--- /dev/null
+++ b/bob/bio/base/test/test_commands.py
@@ -0,0 +1,159 @@
+'''Tests for bob.measure scripts'''
+
+import sys
+import filecmp
+import click
+from click.testing import CliRunner
+import pkg_resources
+from ..script import commands
+
+def test_metrics():
+ dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.txt')
+ runner = CliRunner()
+ result = runner.invoke(commands.metrics, [dev1])
+ with runner.isolated_filesystem():
+ with open('tmp', 'w') as f:
+ f.write(result.output)
+ assert result.exit_code == 0
+ dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-5col.txt')
+ test1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-4col.txt')
+ test2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-5col.txt')
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['--test', dev1, test1, dev2, test2]
+ )
+ with open('tmp', 'w') as f:
+ f.write(result.output)
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
+ )
+ assert result.exit_code == 0
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
+ )
+ assert result.exit_code == 0
+
+def test_roc():
+ dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.txt')
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.roc, ['--output','test.pdf',dev1])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+ dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-5col.txt')
+ test1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-4col.txt')
+ test2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-5col.txt')
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.roc, ['--test', '--split', '--output',
+ 'test.pdf',
+ dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.roc, ['--test', '--output',
+ 'test.pdf', '--titles', 'A,B',
+ dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+
+def test_det():
+ dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.txt')
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.det, [dev1])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+ dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-5col.txt')
+ test1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-4col.txt')
+ test2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-5col.txt')
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.det, ['--test', '--split', '--output',
+ 'test.pdf', '--titles', 'A,B',
+ dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.det, ['--test', '--output',
+ 'test.pdf',
+ dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+def test_epc():
+ dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.txt')
+ test1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-4col.txt')
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.epc, [dev1, test1])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+ dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.tar.gz')
+ test2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-5col.txt')
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.epc, ['--output', 'test.pdf',
+ '--titles', 'A,B',
+ dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+def test_hist():
+ dev1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-4col.txt')
+ dev2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/dev-5col.txt')
+ test1 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-4col.txt')
+ test2 = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/test-5col.txt')
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.hist, [dev1])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
+ 'HISTO.pdf', '-b', 30,
+ dev1, dev2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
+ 'HISTO.pdf', '-b', 30, '-F',
+ 3, dev1, test1, dev2, test2])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0
diff --git a/setup.py b/setup.py
index d33d06d3..cd3ace39 100644
--- a/setup.py
+++ b/setup.py
@@ -73,7 +73,6 @@ setup(
'verify.py = bob.bio.base.script.verify:main',
'resources.py = bob.bio.base.script.resources:resources',
'databases.py = bob.bio.base.script.resources:databases',
- 'evaluate.py = bob.bio.base.script.evaluate:main',
'collect_results.py = bob.bio.base.script.collect_results:main',
'grid_search.py = bob.bio.base.script.grid_search:main',
'preprocess.py = bob.bio.base.script.preprocess:main',
@@ -139,6 +138,12 @@ setup(
# bob bio scripts
'bob.bio.cli': [
'annotate = bob.bio.base.script.annotate:annotate',
+ 'evaluate = bob.bio.base.script.evaluate:evaluate',
+ 'metrics = bob.bio.base.script.commands:metrics',
+ 'roc = bob.bio.base.script.commands:roc',
+ 'det = bob.bio.base.script.commands:det',
+ 'epc = bob.bio.base.script.commands:epc',
+ 'hist = bob.bio.base.script.commands:hist',
],
# annotators
--
GitLab