Commit 02ca511f authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Merge branch 'theo-cli' into 'theo'

generic plotting script for bob measure

See merge request !52
parents a2ebcf46 2b95ebd9
Pipeline #19184 passed with stage
in 42 minutes and 18 seconds
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
==== ===================
.. Development dev-1
==== ===================
FMR 6.263% (31/495)
FNMR 6.208% (28/451)
FAR 5.924%
FRR 11.273%
HTER 8.599%
==== ===================
[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
==== =================== ===============
.. Development dev-1 Test test-1
==== =================== ===============
FMR 6.263% (31/495) 5.637% (27/479)
FNMR 6.208% (28/451) 6.131% (29/473)
FAR 5.924% 5.366%
FRR 11.273% 10.637%
HTER 8.599% 8.001%
==== =================== ===============
[Min. criterion: EER] Threshold on Development set `dev-2`: 1.652567e-02
==== =================== ===============
.. Development dev-2 Test test-2
==== =================== ===============
FMR 4.591% (23/501) 3.333% (16/480)
FNMR 4.484% (20/446) 7.006% (33/471)
FAR 4.348% 3.170%
FRR 9.547% 11.563%
HTER 6.947% 7.367%
==== =================== ===============
......@@ -41,6 +41,34 @@ def split(filename):
the first column containing -1 or 1 (i.e. negative or
positive) and the second the scores
(float).'''.format(filename))
return None, None
raise
return (scores[numpy.where(neg_pos == -1)],
scores[numpy.where(neg_pos == 1)])
def split_files(filenames):
"""split_files
Parse a list of files using :py:func:`split`
Parameters
----------
filenames :
:any:`list`: A list of file paths
Returns
-------
:any:`list`: A list of tuples, where each tuple contains the
``negative`` and ``positive`` scores for one probe of the database. Both
``negatives`` and ``positives`` can be either an 1D
:py:class:`numpy.ndarray` of type ``float``, or ``None``.
"""
if filenames is None:
return None
res = []
for file_path in filenames:
try:
res.append(split(file_path))
except:
raise
return res
......@@ -89,9 +89,9 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
from . import roc as calc
out = calc(negatives, positives, npoints)
if not CAR:
return pyplot.plot(100.0 * out[0, :], 100.0 * out[1, :], **kwargs)
return pyplot.plot(out[0, :], out[1, :], **kwargs)
else:
return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs)
return pyplot.semilogx(out[0, :],(1 - out[1, :]), **kwargs)
def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
......@@ -142,7 +142,7 @@ def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
from matplotlib import pyplot
from . import roc_for_far as calc
out = calc(negatives, positives, far_values)
return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs)
return pyplot.semilogx(out[0, :], (1 - out[1, :]), **kwargs)
def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
......@@ -260,7 +260,7 @@ def epc(dev_negatives, dev_positives, test_negatives, test_positives,
return pyplot.plot(out[0, :], 100.0 * out[1, :], **kwargs)
def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs):
def det(negatives, positives, npoints=100, **kwargs):
"""Plots Detection Error Trade-off (DET) curve as defined in the paper:
Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
......@@ -381,9 +381,9 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs):
pticks = [ppndf(float(v)) for v in desiredTicks]
ax = pyplot.gca() # and finally we set our own tick marks
ax.set_xticks(pticks)
ax.set_xticklabels(desiredLabels, size=axisfontsize)
ax.set_xticklabels(desiredLabels)
ax.set_yticks(pticks)
ax.set_yticklabels(desiredLabels, size=axisfontsize)
ax.set_yticklabels(desiredLabels)
return retval
......@@ -481,9 +481,9 @@ def cmc(cmc_scores, logx=True, **kwargs):
out = calc(cmc_scores)
if logx:
pyplot.semilogx(range(1, len(out) + 1), out * 100, **kwargs)
pyplot.semilogx(range(1, len(out) + 1), out, **kwargs)
else:
pyplot.plot(range(1, len(out) + 1), out * 100, **kwargs)
pyplot.plot(range(1, len(out) + 1), out, **kwargs)
return len(out)
......
''' Click commands for ``bob.measure`` '''
import click
from .. import load
from . import figure
from . import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
@common_options.table_option()
@common_options.output_log_metric_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kwargs):
"""Prints a table that contains FtA, FAR, FRR, FMR, FMNR, HTER for a given
threshold criterion (eer or hter).
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
Resulting table format can be changed using the `--tablefmt`.
Examples:
$ bob measure metrics dev-scores
$ bob measure metrics -l results.txt dev-scores1 eval-scores1
$ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
"""
process = figure.Metrics(ctx, scores, evaluation, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.eval_option()
@common_options.points_curve_option()
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, evaluation, **kwargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
computed using :py:func:`bob.measure.roc`.
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
Examples:
$ bob measure roc dev-scores
$ bob measure roc dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
"""
process = figure.Roc(ctx, scores, evaluation, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.points_curve_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, evaluation, **kwargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
Examples:
$ bob measure det dev-scores
$ bob measure det dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure det -o my_det.pdf dev-scores1 eval-scores1
"""
process = figure.Det(ctx, scores, evaluation, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.title_option()
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.figsize_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
"""Plot EPC (expected performance curve):
plots the error rate on the eval set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost
in [0; 1] of FPR and FNR when calculating the threshold.
You need to provide one or more development score and eval file(s)
for each experiment.
Examples:
$ bob measure epc dev-scores eval-scores
$ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
"""
process = figure.Epc(ctx, scores, True, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option()
@common_options.title_option()
@common_options.titles_option()
@common_options.figsize_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
computed from dev-scores. If you want to display dev-scores distributions
as well, use ``--show-dev`` option.
Examples:
$ bob measure hist dev-scores
$ bob measure hist dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1
"""
process = figure.Hist(ctx, scores, evaluation, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.table_option()
@common_options.eval_option()
@common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.n_bins_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, evaluation, **kwargs):
'''Runs error analysis on score sets
\b
1. Computes the threshold using either EER or min. HTER criteria on
development set scores
2. Applies the above threshold on evaluation set scores to compute the HTER, if a
eval-score set is provided
3. Reports error rates on the console
4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
file
You need to provide 2 score files for each biometric system in this order:
\b
* development scores
* evaluation scores
Examples:
$ bob measure evaluate dev-scores
$ bob measure evaluate scores-dev1 scores-eval1 scores-dev2
scores-eval2
$ bob measure evaluate /path/to/sys-{1,2,3}/scores-{dev,eval}
$ bob measure evaluate -l metrics.txt -o my_plots.pdf dev-scores eval-scores
'''
# first time erase if existing file
ctx.meta['open_mode'] = 'w'
click.echo("Computing metrics with EER...")
ctx.meta['criter'] = 'eer' # no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
# second time, appends the content
ctx.meta['open_mode'] = 'a'
click.echo("Computing metrics with HTER...")
ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log'])
# avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False
if evaluation:
click.echo("Starting evaluate with dev and eval scores...")
else:
click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...")
# set axes limits for ROC
ctx.forward(roc) # use class defaults plot settings
click.echo("Computing DET...")
ctx.forward(det) # use class defaults plot settings
if evaluation:
click.echo("Computing EPC...")
ctx.forward(epc) # use class defaults plot settings
# the last one closes the file
ctx.meta['closef'] = True
click.echo("Computing score histograms...")
ctx.meta['criter'] = 'eer' # no criterion passed in evaluate
ctx.forward(hist)
click.echo("Evaluate successfully completed!")
click.echo("[plots] => %s" % (ctx.meta['output']))
'''Stores click common options for plots'''
import logging
import click
from click.types import INT, FLOAT
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from bob.extension.scripts.click_helper import (bool_option, list_float_option)
LOGGER = logging.getLogger(__name__)
def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
"""Get the argument for scores, and add `dev-scores` and `eval-scores` in
the context when `--evaluation` flag is on (default)
Parameters
----------
eval_mandatory :
If evaluation files are mandatory
min_len :
The min lenght of inputs files that are needed. If eval_mandatory is
True, this quantity is multiplied by 2.
Returns
-------
callable
A decorator to be used for adding score arguments for click commands
"""
def custom_scores_argument(func):
def callback(ctx, param, value):
length = len(value)
min_arg = min_len or 1
ctx.meta['min_arg'] = min_arg
if length < min_arg:
raise click.BadParameter(
'You must provide at least %d score files' % min_arg,
ctx=ctx
)
else:
ctx.meta['scores'] = value
step = 1
if eval_mandatory or ctx.meta['evaluation']:
step = 2
if (length % (min_arg * 2)) != 0:
pref = 'T' if eval_mandatory else \
('When `--evaluation` flag is on t')
raise click.BadParameter(
'%sest-score(s) must '
'be provided along with dev-score(s). '
'You must provide at least %d score files.' \
% (pref, min_arg * 2), ctx=ctx
)
for arg in range(min_arg):
ctx.meta['dev_scores_%d' % arg] = [
value[i] for i in range(arg * step, length,
min_arg * step)
]
if step > 1:
ctx.meta['eval_scores_%d' % arg] = [
value[i] for i in range((arg * step + 1),
length, min_arg * step)
]
ctx.meta['n_sys'] = len(ctx.meta['dev_scores_0'])
if 'titles' in ctx.meta and \
len(ctx.meta['titles']) != ctx.meta['n_sys']:
raise click.BadParameter(
'#titles not equal to #sytems', ctx=ctx
)
return value
return click.argument(
'scores', type=click.Path(exists=True),
callback=callback, **kwargs
)(func)
return custom_scores_argument
def eval_option(**kwargs):
'''Get option flag to say if eval-scores are provided'''
return bool_option(
'evaluation', 'e', 'If set, evaluation scores must be provided',
dflt=True
)
def sep_dev_eval_option(dflt=True, **kwargs):
'''Get option flag to say if dev and eval plots should be in different
plots'''
return bool_option(
'split', 's','If set, evaluation and dev curve in different plots',
dflt
)
def cmc_option(**kwargs):
'''Get option flag to say if cmc scores'''
return bool_option('cmc', 'C', 'If set, CMC score files are provided')
def semilogx_option(dflt=False, **kwargs):
'''Option to use semilog X-axis'''
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt)
def show_dev_option(dflt=False, **kwargs):
'''Option to tell if should show dev histo'''
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt)
def print_filenames_option(dflt=True, **kwargs):
'''Option to tell if filenames should be in the title'''
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt)
def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
def custom_layout_option(func):
def callback(ctx, param, value):
ctx.meta['clayout'] = value
plt.rcParams['figure.constrained_layout.use'] = value
return value
return click.option(
'-Y', '--clayout/--no-clayout', default=dflt, show_default=True,
help='(De)Activate constrained layout',
callback=callback, **kwargs)(func)
return custom_layout_option
def axes_val_option(dflt=None, **kwargs):
''' Option for setting min/max values on axes '''
return list_float_option(
name='axlim', short_name='L',
desc='min/max axes values separated by commas (e.g. ``--axlim '
' 0.1,100,0.1,100``)',
nitems=4, dflt=dflt, **kwargs
)
def thresholds_option(**kwargs):
''' Option to give a list of thresholds '''
return list_float_option(
name='thres', short_name='T',
desc='Given threshold for metrics computations, e.g. '
'0.005,0.001,0.056',
nitems=None, dflt=None, **kwargs
)
def lines_at_option(**kwargs):
'''Get option to draw const far line'''
return list_float_option(
name='lines-at', short_name='la',
desc='If given, draw veritcal lines at the given axis positions',
nitems=None, dflt=None, **kwargs
)
def x_rotation_option(dflt=0, **kwargs):
'''Get option for rotartion of the x axis lables'''
def custom_x_rotation_option(func):
def callback(ctx, param, value):
value = abs(value)
ctx.meta['x_rotation'] = value
return value
return click.option(
'-r', '--x-rotation', type=click.INT, default=dflt, show_default=True,
help='X axis labels ration',
callback=callback, **kwargs)(func)
return custom_x_rotation_option
def cost_option(**kwargs):
'''Get option to get cost for FAR'''
def custom_cost_option(func):
def callback(ctx, param, value):
if value < 0 or value > 1:
raise click.BadParameter("Cost for FAR must be betwen 0 and 1")
ctx.meta['cost'] = value
return value
return click.option(
'-C', '--cost', type=float, default=0.99, show_default=True,
help='Cost for FAR in minDCF',
callback=callback, **kwargs)(func)
return custom_cost_option
def points_curve_option(**kwargs):
'''Get the number of points use to draw curves'''
def custom_points_curve_option(func):
def callback(ctx, param, value):
if value < 2:
raise click.BadParameter(
'Number of points to draw curves must be greater than 1'
, ctx=ctx
)
ctx.meta['points'] = value
return value
return click.option(
'-n', '--points', type=INT, default=100, show_default=True,
help='The number of points use to draw curves in plots',
callback=callback, **kwargs)(func)
return custom_points_curve_option
def n_bins_option(**kwargs):
'''Get the number of bins in the histograms'''
def custom_n_bins_option(func):
def callback(ctx, param, value):
if value is None:
value = 'auto'
elif value < 2:
raise click.BadParameter(
'Number of bins must be greater than 1'
, ctx=ctx
)
ctx.meta['n_bins'] = value
return value
return click.option(
'-b', '--nbins', type=INT, default=None,