Commit c9789352 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Add measure scripts: metrics (HTER, EER), roc, det, epc, hist (score...

Add measure scripts: metrics (HTER, EER), roc, det, epc, hist (score histograms) and evaluate (that generates all the previous at once). Scripts can be called using  with xxx = roc for example. Add tests for those scripts.
parent cb12cf37
Pipeline #17854 failed with stage
in 31 minutes and 19 seconds
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
[Min. criterion: EER] Threshold on Development set `dev-1.txt`: -8.025286e-03
╒══════╤═════════════════════════╕
│ │ Development dev-1.txt │
╞══════╪═════════════════════════╡
│ FMR │ 6.263% (31/495) │
├──────┼─────────────────────────┤
│ FNMR │ 6.208% (28/451) │
├──────┼─────────────────────────┤
│ FAR │ 5.924% │
├──────┼─────────────────────────┤
│ FRR │ 11.273% │
├──────┼─────────────────────────┤
│ HTER │ 8.599% │
╘══════╧═════════════════════════╛
[Min. criterion: EER] Threshold on Development set `dev-1.txt`: -8.025286e-03
╒══════╤═════════════════════════╤═══════════════════╕
│ │ Development dev-1.txt │ Test test-1.txt │
╞══════╪═════════════════════════╪═══════════════════╡
│ FMR │ 6.263% (31/495) │ 5.637% (27/479) │
├──────┼─────────────────────────┼───────────────────┤
│ FNMR │ 6.208% (28/451) │ 6.131% (29/473) │
├──────┼─────────────────────────┼───────────────────┤
│ FAR │ 5.924% │ 5.366% │
├──────┼─────────────────────────┼───────────────────┤
│ FRR │ 11.273% │ 10.637% │
├──────┼─────────────────────────┼───────────────────┤
│ HTER │ 8.599% │ 8.001% │
╘══════╧═════════════════════════╧═══════════════════╛
[Min. criterion: EER] Threshold on Development set `dev-2.txt`: 1.652567e-02
╒══════╤═════════════════════════╤═══════════════════╕
│ │ Development dev-2.txt │ Test test-2.txt │
╞══════╪═════════════════════════╪═══════════════════╡
│ FMR │ 4.591% (23/501) │ 3.333% (16/480) │
├──────┼─────────────────────────┼───────────────────┤
│ FNMR │ 4.484% (20/446) │ 7.006% (33/471) │
├──────┼─────────────────────────┼───────────────────┤
│ FAR │ 4.348% │ 3.170% │
├──────┼─────────────────────────┼───────────────────┤
│ FRR │ 9.547% │ 11.563% │
├──────┼─────────────────────────┼───────────────────┤
│ HTER │ 6.947% │ 7.367% │
╘══════╧═════════════════════════╧═══════════════════╛
'''Stores click common options for plots'''
import pkg_resources # to make sure bob gets imported properly
import logging
import click
from click.types import INT, FLOAT, Choice, File
......@@ -9,104 +10,303 @@ from bob.extension.scripts.click_helper import verbosity_option
logger = logging.getLogger(__name__)
def plot_options(f):
# more import options go down the list here.
f = click.pass_context(f)
f = verbosity_option()(f)
f = click.option(
'--style', multiple=True, type=click.types.Choice(plt.style.available),
help='The matplotlib style to use for plotting. You can provide '
'multiple styles by repeating this option')(f)
f = click.option(
'--titles', help='The title for each system comma separated. '
'Example: --titles ISV,CNN')(f)
f = click.option(
'--top', type=FLOAT,
help='To give to ``plt.subplots_adjust(top=top)``. If given, first '
'plt.tight_layout is called. If you want to tight_layout to be called,'
' then you need to provide this option.')(f)
f = click.option(
'--legend-ncol', default=3, show_default=True,
type=INT,
help='The number of columns of the legend layout.')(f)
f = click.option(
'--figsize', help='If given, will run '
'``plt.figure(figsize=figsize)(f)``. Example: --fig-size 4,6')(f)
# f = click.option(
# '--y2-label',
# help='The id of figures which should have y2_label separated by '
# 'comma. For example ``--y2-label 1,2,4``.')(f)
f = click.option(
'--y1-label',
help='The id of figures which should have y1_label separated by '
'comma. For example ``--y1-label 1,2,4``.')(f)
f = click.option(
'--x-label',
help='The id of figures which should have x_label separated by '
'comma. For example ``--x-label 1,2,4``.')(f)
f = click.option(
'--subplot', type=INT, default=111,
show_default=True, help='The order of subplots.')(f)
f = click.option(
'-o', '--output', type=File(mode='wb'),
default='plots.pdf', show_default=True,
help='The file to save the plots in.')(f)
return f
def normalize_options(ctx, n_systems, output, subplot, style, x_label,
y1_label, figsize, legend_ncol, top, titles,
y2_label=None):
if style:
plt.style.use(style)
ctx.meta['output'] = output
ctx.meta['PdfPages'] = PdfPages(output)
ctx.meta['x_label'] = x_label if x_label is None else \
[int(x) for x in x_label.split(',')]
ctx.meta['y1_label'] = y1_label if y1_label is None else \
[int(x) for x in y1_label.split(',')]
ctx.meta['y2_label'] = y2_label if y2_label is None else \
[int(x) for x in y2_label.split(',')]
ctx.meta['subplot'] = subplot
nrows = subplot // 10
nrows, ncols = divmod(nrows, 10)
logger.debug('Got %d, %d for nrows and ncols', nrows, ncols)
ctx.meta['nrows_ncols'] = nrows, ncols
ctx.meta['figsize'] = figsize if figsize is None else \
[float(x) for x in figsize.split(',')]
plt.figure(figsize=ctx.meta['figsize'])
ctx.meta['legend_ncol'] = legend_ncol
ctx.meta['top'] = top
ctx.meta['titles'] = titles if titles is None else titles.split(',')
nrows, ncols = ctx.meta['nrows_ncols']
if nrows * ncols < n_systems:
logger.error("The number of subplots is smaller than the number of "
"systems. I will plot one system a column. Use --subplot "
"to remove this error.")
nrows, ncols = 1, n_systems
ctx.meta['nrows'], ctx.meta['ncols'] = nrows, ncols
ctx.meta['titles'] = ctx.meta['titles'] or [None] * n_systems
# Try to automatically figure out where to place labels
# x_label should be True if row == -1
# y1_label should be True if col == 0
# y2_label should be True if col == -1
ctx.meta['x_label'] = ctx.meta['x_label'] or \
[x for x in range(1, n_systems + 1)
if ((x - 1) // ncols) == (nrows - 1)]
ctx.meta['y1_label'] = ctx.meta['y1_label'] or \
[x for x in range(1, n_systems + 1)
if ((x - 1) % ncols) == 0]
ctx.meta['y2_label'] = ctx.meta.get('y2_label', None) or \
[x for x in range(1, n_systems + 1)
if ((x - 1) % ncols) == (ncols - 1)]
return ctx
def scores_argument(test_mandatory=False, **kwargs):
'''Get the argument for scores, and add `dev-scores` and `test-scores` in
the context if `--test` flag is on (default `--no-test`).'''
def custom_scores_argument(func):
def callback(ctx, param, value):
length = len(value)
if length < 1:
raise click.BadParameter('No scores provided', ctx=ctx)
else:
ctx.meta['scores'] = value
if test_mandatory or ctx.meta['test']:
if (length % 2) != 0:
pref = 'T' if test_mandatory else ('When `--test` flag'
' is on t')
raise click.BadParameter(
'%sest-score(s) must '
'be provided along with dev-score(s)' % pref, ctx=ctx)
else:
ctx.meta['dev-scores'] = [value[i] for i in
range(length) if not i % 2]
ctx.meta['test-scores'] = [value[i] for i in
range(length) if i % 2]
ctx.meta['n_sys'] = len(ctx.meta['test-scores'])
return value
return click.argument('scores', callback=callback, **kwargs)(func)
return custom_scores_argument
def test_option(**kwargs):
'''Get option flag to say if test-scores are provided'''
def custom_test_option(func):
def callback(ctx, param, value):
ctx.meta['test'] = value
return value
return click.option(
'-t', '--test/--no-test', default=False,
help='If set, test scores must be provided',
callback=callback, is_eager=True ,**kwargs)(func)
return custom_test_option
def n_sys_option(**kwargs):
'''Get the number of systems to be processed'''
def custom_n_sys_option(func):
def callback(ctx, param, value):
ctx.meta['n_sys'] = value
return value
return click.option(
'--n-sys', type=INT, default=1, show_default=True,
help='The number of systems to be processed',
callback=callback, is_eager=True ,**kwargs)(func)
return custom_n_sys_option
def points_curve_option(**kwargs):
'''Get the number of points use to draw curves'''
def custom_points_curve_option(func):
def callback(ctx, param, value):
if value < 2:
raise click.BadParameter(
'Number of points to draw curves must be greater than 1'
, ctx=ctx
)
ctx.meta['n'] = value
return value
return click.option(
'-n', '--points', type=INT, default=100, show_default=True,
help='The number of points use to draw curves in plots',
callback=callback, **kwargs)(func)
return custom_points_curve_option
def n_bins_option(**kwargs):
'''Get the number of bins in the histograms'''
def custom_n_bins_option(func):
def callback(ctx, param, value):
if value < 2:
raise click.BadParameter(
'Number of bins must be greater than 1'
, ctx=ctx
)
ctx.meta['n_bins'] = value
return value
return click.option(
'-b', '--nbins', type=INT, default=20, show_default=True,
help='The number of bins in the histogram(s)',
callback=callback, **kwargs)(func)
return custom_n_bins_option
@click.option('-n', '--points', type=INT, default=100, show_default=True,
help='Number of points to use in the curves')
def table_option(**kwargs):
'''Get table option for tabulate package
More informnations: https://pypi.python.org/pypi/tabulate
'''
def custom_table_option(func):
def callback(ctx, param, value):
if value is not None:
ctx.meta['tablefmt'] = value
elif 'log' in ctx.meta and ctx.meta['log'] is not None:
value = 'latex'
else:
value = 'fancy_grid'
ctx.meta['tablefmt'] = value
return value
return click.option(
'--tablefmt', type=click.STRING, default=None,
show_default=True, help='Format for table display: `plain`, '
'`simple`, `grid`, (default) `fancy_grid`, `pipe`, `orgtbl`, '
'`jira`, `presto`, `psql`, `rst`, `mediawiki`, `moinmoin`, '
'`youtrack`, `html`, (default with `--log`)`latex`, '
'`latex_raw`, `latex_booktabs`, `textile`',
callback=callback,**kwargs)(func)
return custom_table_option
def output_plot_file_option(default_out='plots.pdf', **kwargs):
'''Get options for output file for plots'''
def custom_output_plot_file_option(func):
def callback(ctx, param, value):
''' Save ouput file and associated pdf in context list,
print the path of the file in the log'''
ctx.meta['output'] = value
ctx.meta['PdfPages'] = PdfPages(value)
logger.debug("Plots will be output in %s", value)
return value
return click.option(
'-o', '--output',
default=default_out, show_default=True,
help='The file to save the plots in.',
callback=callback, **kwargs)(func)
return custom_output_plot_file_option
def output_plot_metric_option(**kwargs):
'''Get options for output file for metrics'''
def custom_output_plot_file_option(func):
def callback(ctx, param, value):
''' Save ouput file and associated pdf in context list,
print the path of the file in the log'''
if value is not None:
logger.debug("Metrics will be output in %s", value)
ctx.meta['log'] = value
return value
return click.option(
'-l', '--log', default=None, type=click.STRING,
help='If provided, computed numbers are written to '
'this file instead of the standard output.',
callback=callback, **kwargs)(func)
return custom_output_plot_file_option
def open_file_mode_option(**kwargs):
'''Get the top option for matplotlib'''
def custom_open_file_mode_option(func):
def callback(ctx, param, value):
if value not in ['w', 'a', 'w+', 'a+']:
raise click.BadParameter('Incorrect open file mode')
ctx.meta['open-mode'] = value
return value
return click.option(
'-om', '--open-mode', default='w',
help='File open mode',
callback=callback, **kwargs)(func)
return custom_open_file_mode_option
def criterion_option(**kwargs):
'''Get option flag to tell which criteriom is used (default:eer)'''
def custom_criterion_option(func):
def callback(ctx, param, value):
list_accepted_crit = ['eer', 'hter']
if value not in list_accepted_crit:
raise click.BadParameter('Incorrect value for `--criter`. '
'Must be one of [`%s`]' %
'`, `'.join(list_accepted_crit))
ctx.meta['criter'] = value
return value
return click.option(
'--criter', default='eer', help='Criterion to compute plots and '
'metrics: `eer` (default), `hter`',
callback=callback, is_eager=True ,**kwargs)(func)
return custom_criterion_option
def label_option(name_option='x-label', **kwargs):
'''Get labels options based on the given name.
Parameters:
----------
name_option: str, optional
Name of the label option (e.g. x-lable, y1-label)
'''
def custom_label_option(func):
def callback(ctx, param, value):
''' Get and save labels list in the context list '''
ctx.meta[name_option] = value if value is None else \
[int(i) for i in value.split(',')]
return value
return click.option(
'--' + name_option,
help='The id of figures which should have x_label separated by '
'comma. For example ``--%s 1,2,4``.' % name_option,
callback=callback, **kwargs)(func)
return custom_label_option
def figsize_option(**kwargs):
'''Get option for matplotlib figsize'''
def custom_figsize_option(func):
def callback(ctx, param, value):
ctx.meta['figsize'] = value if value is None else \
[float(x) for x in value.split(',')]
plt.figure(figsize=ctx.meta['figsize'])
return value
return click.option(
'--figsize', help='If given, will run \
``plt.figure(figsize=figsize)(f)``. Example: --fig-size 4,6',
callback=callback, **kwargs)(func)
return custom_figsize_option
def legend_ncols_option(**kwargs):
'''Get the number of columns to set in the legend of the plot'''
def custom_legend_ncols_option(func):
def callback(ctx, param, value):
ctx.meta['legend_ncol'] = value
return value
return click.option(
'--legend-ncol', default=3, show_default=True,
type=INT, help='The number of columns of the legend layout.',
callback=callback, **kwargs)(func)
return custom_legend_ncols_option
def legend_loc_option(**kwargs):
'''Get tthe legend location of the plot'''
def custom_legend_loc_option(func):
def callback(ctx, param, value):
ctx.meta['legend_loc'] = value
return value
return click.option(
'--legend-location', default=0, show_default=True,
type=INT, help='The lengend location code',
callback=callback, **kwargs)(func)
return custom_legend_loc_option
def line_width_option(**kwargs):
'''Get line width option for the plots'''
def custom_line_width_option(func):
def callback(ctx, param, value):
ctx.meta['line_width'] = value
return value
return click.option(
'--line-width',
type=FLOAT, help='The line width of plots',
callback=callback, **kwargs)(func)
return custom_line_width_option
def marker_style_option(**kwargs):
'''Get marker style otpion for the plots'''
def custom_marker_style_option(func):
def callback(ctx, param, value):
ctx.meta['marker_style'] = value
return value
return click.option(
'--marker-style',
type=FLOAT, help='The marker style of the plots',
callback=callback, **kwargs)(func)
return custom_marker_style_option
def top_option(**kwargs):
'''Get the top option for matplotlib'''
def custom_top_option(func):
def callback(ctx, param, value):
ctx.meta['top'] = value
return value
return click.option(
'--top', type=FLOAT,
help='To give to ``plt.subplots_adjust(top=top)``. If given, first'
' plt.tight_layout is called. If you want to tight_layout to be '
'called, then you need to provide this option.',
callback=callback, **kwargs)(func)
return custom_top_option
def titles_option(**kwargs):
'''Get the titles otpion for the different systems'''
def custom_titles_option(func):
def callback(ctx, param, value):
ctx.meta['titles'] = value if value is None else \
value.split(',')
return value
return click.option(
'--titles', help='The title for each system comma separated. '
'Example: --titles ISV,CNN',
callback=callback, **kwargs)(func)
return custom_titles_option
def style_option(**kwargs):
'''Get option for matplotlib style'''
def custom_style_option(func):
def callback(ctx, param, value):
ctx.meta['style'] = value
plt.style.use(value)
return value
return click.option(
'--style', multiple=True, type=click.types.Choice(plt.style.available),
help='The matplotlib style to use for plotting. You can provide '
'multiple styles by repeating this option',
callback=callback, **kwargs)(func)
return custom_style_option
This diff is collapsed.
......@@ -3,10 +3,12 @@
import click
import pkg_resources
from click_plugins import with_plugins
from click.types import INT, FLOAT, Choice, File
@with_plugins(pkg_resources.iter_entry_points('bob.measure.cli'))
@click.group()
@click.group(chain=True)
def measure():
"""Entry for bob.measure commands."""
pass
'''Tests for bob.measure scripts'''
import os
import filecmp
import tempfile
import subprocess
import click
from click.testing import CliRunner
import bob.io.base.test_utils
from .script import evaluate
def test_metrics():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
result = runner.invoke(evaluate.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
assert filecmp.cmp(test_ref, 'tmp')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['--test', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m2.txt', 'bob.measure')
assert filecmp.cmp(test_ref, 'tmp')
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, dev2]
)
assert result.exit_code == 0
def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--output','test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_det():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_epc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, ['--output', 'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_hist():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, [dev1])
if result.output: