Commit 3b18e032 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Implement set of commands for bio.base based on bob.measure

parent 816862e7
Pipeline #18430 failed with stage
in 21 minutes and 57 seconds
......@@ -354,7 +354,6 @@ def cmc(filename, ncolumns=None):
assert ncolumns == 5
return cmc_five_column(filename)
def load_score(filename, ncolumns=None, minimal=False, **kwargs):
"""Load scores using numpy.loadtxt and return the data as a numpy array.
......@@ -419,6 +418,37 @@ def load_score(filename, ncolumns=None, minimal=False, **kwargs):
score_lines = numpy.array(score_lines, new_dtype)
return score_lines
def load_files(filenames, func_load):
"""Load a list of score files and return a list of tuples of (neg, pos)
Parameters
----------
filenames : :any:`list`
list of file paths
func_load :
function that can read files in the list
Returns
-------
:any:`list`: [(neg,pos)] A list of tuples, where each tuple contains the
``negative`` and ``positive`` sceach system/probee.
"""
if filenames is None:
return None
res = []
for filepath in filenames:
try:
tmp = func_load(filepath)
if isinstance(tmp, list):
res += func_load(filepath)
else:
res.append(tmp)
except:
raise
return res
def get_negatives_positives(score_lines):
"""Take the output of load_score and return negatives and positives. This
......
''' Click commands for ``bob.bio.base`` '''
import click
from ..score import load
from bob.measure.script import figure
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, test, **kargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Resulting table format can be changer using the `--tablefmt`. Default
formats are `rst` when output in the terminal and `latex` when
written in a log file (see `--log`)
Examples:
$ bob bio metrics dev-scores
$ bob bio metrics --test -l results.txt dev-scores1 test-scores1
$ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
"""
process = figure.Metrics(ctx, scores, test, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.test_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, test, **kargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
computed using :py:func:`bob.measure.roc`.
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio roc dev-scores
$ bob bio roc --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob bio roc --test -o my_roc.pdf dev-scores1 test-scores1
"""
process = figure.Roc(ctx, scores, test, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.test_option()
@common_options.axis_fontsize_option(dflt=6)
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, test, **kargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio det dev-scores
$ bob bio det --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob bio det --test -o my_det.pdf dev-scores1 test-scores1
"""
process = figure.Det(ctx, scores, test, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.axis_fontsize_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kargs):
"""Plot EPC (expected performance curve):
plots the error rate on the test set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost
in [0; 1] of FPR and FNR when calculating the threshold.
You need provide one or more development score and test file(s)
for each experiment. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio epc dev-scores test-scores
$ bob bio epc -o my_epc.pdf dev-scores1 test-scores1
"""
process = figure.Epc(ctx, scores, True, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.test_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.
Examples:
$ bob bio hist dev-scores
$ bob bio hist --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob bio hist --test --criter hter dev-scores1 test-scores1
"""
process = figure.Hist(ctx, scores, test, FUNC_SPLIT)
process.run()
'''Tests for bob.measure scripts'''
import sys
import filecmp
import click
from click.testing import CliRunner
import pkg_resources
from ..script import commands
def test_metrics():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
assert result.exit_code == 0
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--test', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
)
assert result.exit_code == 0
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output','test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--test', '--split', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--test', '--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_det():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.det, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--test', '--split', '--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_epc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.tar.gz')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, ['--output', 'test.pdf',
'--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
'HISTO.pdf', '-b', 30,
dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F',
3, dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -73,7 +73,6 @@ setup(
'verify.py = bob.bio.base.script.verify:main',
'resources.py = bob.bio.base.script.resources:resources',
'databases.py = bob.bio.base.script.resources:databases',
'evaluate.py = bob.bio.base.script.evaluate:main',
'collect_results.py = bob.bio.base.script.collect_results:main',
'grid_search.py = bob.bio.base.script.grid_search:main',
'preprocess.py = bob.bio.base.script.preprocess:main',
......@@ -139,6 +138,12 @@ setup(
# bob bio scripts
'bob.bio.cli': [
'annotate = bob.bio.base.script.annotate:annotate',
'evaluate = bob.bio.base.script.evaluate:evaluate',
'metrics = bob.bio.base.script.commands:metrics',
'roc = bob.bio.base.script.commands:roc',
'det = bob.bio.base.script.commands:det',
'epc = bob.bio.base.script.commands:epc',
'hist = bob.bio.base.script.commands:hist',
],
# annotators
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment