Commit 7be56833 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

reshape code so that the figures can be re-used in other modules (class based...

reshape code so that the figures can be re-used in other modules (class based plotting, add input loading function to read inputs files), split the code in different files (utils.py for utilities, figure.py for plotting related classes, commands.py for click commands). Change some click command defaults (nbins:  instead of 20, tablefmt:  instead of ), add click functionalities for a greater control of the plots (e.g. min/max axes values) and a nicer default visual. Fix the documentation and modify user guide according to the changes.
parent b349ddaf
Pipeline #18381 failed with stage
in 26 minutes and 37 seconds
[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
╒══════╤═════════════════════╕
│ │ Development dev-1 │
╞══════╪═════════════════════╡
│ FMR │ 6.263% (31/495) │
├──────┼─────────────────────┤
│ FNMR │ 6.208% (28/451) │
├──────┼─────────────────────┤
│ FAR │ 5.924% │
├──────┼─────────────────────┤
│ FRR │ 11.273% │
├──────┼─────────────────────┤
│ HTER │ 8.599% │
╘══════╧═════════════════════╛
==== ===================
.. Development dev-1
==== ===================
FMR 6.263% (31/495)
FNMR 6.208% (28/451)
FAR 5.924%
FRR 11.273%
HTER 8.599%
==== ===================
[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
╒══════╤═════════════════════╤═════════════════╕
│ │ Development dev-1 │ Test test-1 │
╞══════╪═════════════════════╪═════════════════╡
│ FMR │ 6.263% (31/495) │ 5.637% (27/479) │
├──────┼─────────────────────┼─────────────────┤
│ FNMR │ 6.208% (28/451) │ 6.131% (29/473) │
├──────┼─────────────────────┼─────────────────┤
│ FAR │ 5.924% │ 5.366% │
├──────┼─────────────────────┼─────────────────┤
│ FRR │ 11.273% │ 10.637% │
├──────┼─────────────────────┼─────────────────┤
│ HTER │ 8.599% │ 8.001% │
╘══════╧═════════════════════╧═════════════════╛
==== =================== ===============
.. Development dev-1 Test test-1
==== =================== ===============
FMR 6.263% (31/495) 5.637% (27/479)
FNMR 6.208% (28/451) 6.131% (29/473)
FAR 5.924% 5.366%
FRR 11.273% 10.637%
HTER 8.599% 8.001%
==== =================== ===============
[Min. criterion: EER] Threshold on Development set `dev-2`: 1.652567e-02
╒══════╤═════════════════════╤═════════════════╕
│ │ Development dev-2 │ Test test-2 │
╞══════╪═════════════════════╪═════════════════╡
│ FMR │ 4.591% (23/501) │ 3.333% (16/480) │
├──────┼─────────────────────┼─────────────────┤
│ FNMR │ 4.484% (20/446) │ 7.006% (33/471) │
├──────┼─────────────────────┼─────────────────┤
│ FAR │ 4.348% │ 3.170% │
├──────┼─────────────────────┼─────────────────┤
│ FRR │ 9.547% │ 11.563% │
├──────┼─────────────────────┼─────────────────┤
│ HTER │ 6.947% │ 7.367% │
╘══════╧═════════════════════╧═════════════════╛
==== =================== ===============
.. Development dev-2 Test test-2
==== =================== ===============
FMR 4.591% (23/501) 3.333% (16/480)
FNMR 4.484% (20/446) 7.006% (33/471)
FAR 4.348% 3.170%
FRR 9.547% 11.563%
HTER 6.947% 7.367%
==== =================== ===============
......@@ -29,6 +29,9 @@ def split(filename):
This array contains the list of negative scores
positives: 1D :py:class:`numpy.ndarray` of type float
def __init__(self):
pass
This array contains the list of positive scores
"""
......@@ -41,6 +44,34 @@ def split(filename):
the first column containing -1 or 1 (i.e. negative or
positive) and the second the scores
(float).'''.format(filename))
return (None, None)
raise
return (scores[numpy.where(neg_pos == -1)],
scores[numpy.where(neg_pos == 1)])
def split_files(filenames):
"""split_files
Parse a list of files using :py:func:`split`
Parameters
----------
filenames :
:any:`list`: A list of file paths
Returns
-------
:any:`list`: A list of tuples, where each tuple contains the
``negative`` and ``positive`` scores for one probe of the database. Both
``negatives`` and ``positives`` can be either an 1D
:py:class:`numpy.ndarray` of type ``float``, or ``None``.
"""
if filenames is None:
return None
res = []
for file_path in filenames:
try:
res.append(split(file_path))
except:
raise
return res
......@@ -89,9 +89,9 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
from . import roc as calc
out = calc(negatives, positives, npoints)
if not CAR:
return pyplot.plot(100.0 * out[0, :], 100.0 * out[1, :], **kwargs)
return pyplot.plot(out[0, :], out[1, :], **kwargs)
else:
return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs)
return pyplot.semilogx(out[0, :],(1 - out[1, :]), **kwargs)
def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
......
import numpy
def remove_nan(scores):
"""removes the NaNs from the scores"""
nans = numpy.isnan(scores)
sum_nans = sum(nans)
total = len(scores)
if sum_nans > 0:
logger.warning('Found {} NaNs in {} scores'.format(sum_nans, total))
return scores[numpy.where(~nans)], sum_nans, total
def get_fta(scores):
"""calculates the Failure To Acquire (FtA) rate"""
fta_sum, fta_total = 0, 0
neg, sum_nans, total = remove_nan(scores[0])
fta_sum += sum_nans
fta_total += total
pos, sum_nans, total = remove_nan(scores[1])
fta_sum += sum_nans
fta_total += total
return ((neg, pos), fta_sum / fta_total)
''' Click commands for ``bob.measure`` '''
import click
from .. import load
from . import figure
from . import common_options
from bob.extension.scripts.click_helper import verbosity_option
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, test, **kargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.
Resulting table format can be changer using the `--tablefmt`. Default
formats are `fancy_grid` when output in the terminal and `latex` when
written in a log file (see `--log`)
Examples:
$ bob measure metrics dev-scores
$ bob measure metrics --test -l results.txt dev-scores1 test-scores1
$ bob measure metrics --test {dev,test}-scores1 {dev,test}-scores2
"""
process = figure.Metrics(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.test_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.min_x_axis_val_option()
@common_options.max_x_axis_val_option(dflt=1)
@common_options.min_y_axis_val_option(dflt=0)
@common_options.max_y_axis_val_option(dflt=1)
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, test, **kargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
computed using :py:func:`bob.measure.roc`.
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.
Examples:
$ bob measure roc dev-scores
$ bob measure roc --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob measure roc --test -o my_roc.pdf dev-scores1 test-scores1
"""
process = figure.Roc(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.test_option()
@common_options.min_x_axis_val_option(dflt=0.01)
@common_options.max_x_axis_val_option(dflt=95)
@common_options.min_y_axis_val_option(dflt=0.01)
@common_options.max_y_axis_val_option(dflt=95)
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, test, **kargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.
Examples:
$ bob measure det dev-scores
$ bob measure det --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob measure det --test -o my_det.pdf dev-scores1 test-scores1
"""
process = figure.Det(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.axis_fontsize_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kargs):
"""Plot EPC (expected performance curve):
plots the error rate on the test set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost β
∈ [0; 1] of FPR and FNR when calculating the threshold.
You need provide one or more development score and test file(s)
for each experiment.
Examples:
$ bob measure epc dev-scores test-scores
$ bob measure epc -o my_epc.pdf dev-scores1 test-scores1
"""
process = figure.Epc(ctx, scores, True, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.test_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
You need provide one or more development score file(s) for each experiment.
You can also provide test files along with dev files but the flag `--test`
is required in that case.
Examples:
$ bob measure hist dev-scores
$ bob measure hist --test dev-scores1 test-scores1 dev-scores2
test-scores2
$ bob measure hist --test --criter hter dev-scores1 test-scores1
"""
process = figure.Hist(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.table_option()
@common_options.test_option()
@common_options.output_plot_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, test, **kargs):
'''Runs error analysis on score sets
1. Computes the threshold using either EER or min. HTER criteria on
development set scores
2. Applies the above threshold on test set scores to compute the HTER, if a
test-score set is provided
3. Reports error rates on the console
4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
file (unless --no-plot is passed)
You need to provide 2 score files for each biometric system in this order:
\b
* development scores
* evaluation scores
Examples:
$ bob measure evaluate dev-scores
$ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores
'''
#first time erase if existing file
click.echo("Computing metrics with EER...")
ctx.meta['criter'] = 'eer' #no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, test=test)
#second time, appends the content
click.echo("Computing metrics with HTER...")
ctx.meta['criter'] = 'hter' #no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, test=test)
if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log'])
#avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False
if test:
click.echo("Starting evaluate with dev and test scores...")
else:
click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...")
ctx.forward(roc)
click.echo("Computing DET...")
ctx.forward(det)
if test:
click.echo("Computing EPC...")
ctx.forward(epc)
#the last one closes the file
ctx.meta['closef'] = True
click.echo("Computing score histograms...")
ctx.forward(hist)
click.echo("Evaluate successfully completed!")
click.echo("[plots] => %s" % (ctx.meta['output']))
'''Stores click common options for plots'''
import math
import pkg_resources # to make sure bob gets imported properly
import logging
import click
......@@ -57,7 +58,7 @@ def test_option(**kwargs):
'-t', '--test/--no-test', default=False,
help='If set, test scores must be provided',
show_default=True,
callback=callback, is_eager=True ,**kwargs)(func)
callback=callback, is_eager=True , **kwargs)(func)
return custom_test_option
def sep_dev_test_option(**kwargs):
......@@ -70,9 +71,112 @@ def sep_dev_test_option(**kwargs):
return click.option(
'-s', '--split/--no-split', default=True, show_default=True,
help='If set, test and dev curve in different plots',
callback=callback, is_eager=True,**kwargs)(func)
callback=callback, is_eager=True, **kwargs)(func)
return custom_sep_dev_test_option
def semilogx_option(dflt= False, **kwargs):
'''Option to use semilog X-axis'''
def custom_semilogx_option(func):
def callback(ctx, param, value):
ctx.meta['semilogx'] = value
return value
return click.option(
'--semilogx/--std-x', default=dflt, show_default=True,
help='If set, use semilog on X axis',
callback=callback, **kwargs)(func)
return custom_semilogx_option
def min_x_axis_val_option(dflt=1e-4, **kwargs):
'''Get option for min value on x axis'''
def custom_min_x_axis_val_option(func):
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_x'] = value
return value
return click.option(
'--min-x', type=float, default=dflt, show_default=True,
help='Min value display on X axis',
callback=callback, **kwargs)(func)
return custom_min_x_axis_val_option
def max_x_axis_val_option(dflt=1.0, **kwargs):
'''Get option for max value on x axis'''
def custom_max_x_axis_val_option(func):
def callback(ctx, param, value):
ctx.meta['max_x'] = value
return value
return click.option(
'--max-x', type=float, default=dflt, show_default=True,
help='Max value display on X axis',
callback=callback, **kwargs)(func)
return custom_max_x_axis_val_option
def min_y_axis_val_option(dflt=1e-4, **kwargs):
'''Get option for min value on y axis'''
def custom_min_y_axis_val_option(func):
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_y'] = value
return value
return click.option(
'--min-y', type=float, default=dflt, show_default=True,
help='Min value display on Y axis',
callback=callback, **kwargs)(func)
return custom_min_y_axis_val_option
def max_y_axis_val_option(dflt=1.0, **kwargs):
'''Get option for max value on y axis'''
def custom_max_y_axis_val_option(func):
def callback(ctx, param, value):
ctx.meta['max_y'] = value
return value
return click.option(
'--max-y', type=float, default=dflt, show_default=True,
help='Max value display on Y axis',
callback=callback, **kwargs)(func)
return custom_max_y_axis_val_option
def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size'''
def custom_axis_fontsize_option(func):
def callback(ctx, param, value):
value = abs(value)
ctx.meta['fontsize'] = value
return value
return click.option(
'-F', '--fontsize', type=click.INT, default=dflt, show_default=True,
help='Axis fontsize',
callback=callback, **kwargs)(func)
return custom_axis_fontsize_option
def x_rotation_option(dflt=0, **kwargs):
'''Get option for rotartion of the x axis lables'''
def custom_x_rotation_option(func):
def callback(ctx, param, value):
value = abs(value)
ctx.meta['x_rotation'] = value
return value
return click.option(
'-r', '--x-rotation', type=click.INT, default=dflt, show_default=True,
help='X axis labels ration',
callback=callback, **kwargs)(func)
return custom_x_rotation_option
def fmr_line_at_option(**kwargs):
'''Get option to draw const fmr line'''
def custom_fmr_line_at_option(func):
def callback(ctx, param, value):
if value is not None:
value = min(max(0.0, value), 1.0)
ctx.meta['fmr_at'] = value
return value
return click.option(
'-L', '--fmr-at', type=float, default=None, show_default=True,
help='If given, draw a veritcal line for constant FMR on ROC plots',
callback=callback, **kwargs)(func)
return custom_fmr_line_at_option
def n_sys_option(**kwargs):
'''Get the number of systems to be processed'''
......@@ -83,7 +187,7 @@ def n_sys_option(**kwargs):
return click.option(
'--n-sys', type=INT, default=1, show_default=True,
help='The number of systems to be processed',
callback=callback, is_eager=True ,**kwargs)(func)
callback=callback, is_eager=True , **kwargs)(func)
return custom_n_sys_option
def points_curve_option(**kwargs):
......@@ -95,7 +199,7 @@ def points_curve_option(**kwargs):
'Number of points to draw curves must be greater than 1'
, ctx=ctx
)
ctx.meta['n'] = value
ctx.meta['points'] = value
return value
return click.option(
'-n', '--points', type=INT, default=100, show_default=True,
......@@ -107,7 +211,9 @@ def n_bins_option(**kwargs):
'''Get the number of bins in the histograms'''
def custom_n_bins_option(func):
def callback(ctx, param, value):
if value < 2:
if value is None:
value = 'auto'
elif value < 2:
raise click.BadParameter(
'Number of bins must be greater than 1'
, ctx=ctx
......@@ -115,8 +221,8 @@ def n_bins_option(**kwargs):
ctx.meta['n_bins'] = value
return value
return click.option(
'-b', '--nbins', type=INT, default=20, show_default=True,
help='The number of bins in the histogram(s)',
'-b', '--nbins', type=INT, default=None,
help='The number of bins in the histogram(s). Default: `auto`',
callback=callback, **kwargs)(func)
return custom_n_bins_option
......@@ -131,14 +237,14 @@ def table_option(**kwargs):
elif 'log' in ctx.meta and ctx.meta['log'] is not None:
value = 'latex'
else:
value = 'fancy_grid'
value = 'rst'
ctx.meta['tablefmt'] = value
return value
return click.option(
'--tablefmt', type=click.STRING, default=None,
show_default=True, help='Format for table display: `plain`, '
'`simple`, `grid`, (default) `fancy_grid`, `pipe`, `orgtbl`, '
'`jira`, `presto`, `psql`, `rst`, `mediawiki`, `moinmoin`, '
'`simple`, `grid`, `fancy_grid`, `pipe`, `orgtbl`, '
'`jira`, `presto`, `psql`, (default) `rst`, `mediawiki`, `moinmoin`, '
'`youtrack`, `html`, (default with `--log`)`latex`, '
'`latex_raw`, `latex_booktabs`, `textile`',
callback=callback,**kwargs)(func)
......@@ -184,7 +290,7 @@ def open_file_mode_option(**kwargs):
def callback(ctx, param, value):
if value not in ['w', 'a', 'w+', 'a+']:
raise click.BadParameter('Incorrect open file mode')
ctx.meta['open-mode'] = value
ctx.meta['open_mode'] = value
return value
return click.option(
'-om', '--open-mode', default='w',
......
This diff is collapsed.
This diff is collapsed.
'''Tests for bob.measure scripts'''
import sys
import os
import filecmp
import tempfile
import subprocess
import click
from click.testing import CliRunner
import bob.io.base.test_utils
from .script import evaluate
from .script import commands
def test_metrics():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
result = runner.invoke(evaluate.metrics, [dev1])
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -28,7 +25,7 @@ def test_metrics():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['--test', dev1, test1, dev2, test2]
commands.metrics, ['--test', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -40,12 +37,12 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
commands.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, dev2]
commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
)
assert result.exit_code == 0
......@@ -53,7 +50,7 @@ def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--output','test.pdf',dev1])
result = runner.invoke(commands.roc, ['--output','test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -61,7 +58,7 @@ def test_roc():
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--test', '--split', '--output',
result = runner.invoke(commands.roc, ['--test', '--split', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
......@@ -69,7 +66,7 @@ def test_roc():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--test', '--output',
result = runner.invoke(commands.roc, ['--test', '--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
......@@ -81,7 +78,7 @@ def test_det():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, [dev1])
result = runner.invoke(commands.det, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -89,14 +86,14 @@ def test_det():
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, ['--test', '--split', '--output',
result = runner.invoke(commands.det, ['--test', '--split', '--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, ['--test', '--output',
result = runner.invoke(commands.det, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
......@@ -108,14 +105,14 @@ def test_epc():
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, [dev1, test1])
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, ['--output', 'test.pdf',
result = runner.invoke(commands.epc, ['--output', 'test.pdf',
'--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
......@@ -129,13 +126,13 @@ def test_hist():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, [dev1])
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, ['--criter', 'hter','--output',
result = runner.invoke(commands.hist, ['--criter', 'hter','--output',
'HISTO.pdf', '-b', 30,
dev1, dev2])
if result.output:
......@@ -143,7 +140,7 @@ def test_hist():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, ['--criter', 'eer','--output',