Commit ddfd5ceb authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'theo-cli' into 'theo'

Set of click commands for bio base

See merge request !143
parents 863ca73a 859731b6
Pipeline #19336 failed with stage
in 24 minutes and 51 seconds
......@@ -363,7 +363,6 @@ def cmc(filename, ncolumns=None):
assert ncolumns == 5
return cmc_five_column(filename)
def load_score(filename, ncolumns=None, minimal=False, **kwargs):
"""Load scores using numpy.loadtxt and return the data as a numpy array.
......@@ -428,6 +427,33 @@ def load_score(filename, ncolumns=None, minimal=False, **kwargs):
score_lines = numpy.array(score_lines, new_dtype)
return score_lines
def load_files(filenames, func_load):
"""Load a list of score files and return a list of tuples of (neg, pos)
Parameters
----------
filenames : :any:`list`
list of file paths
func_load :
function that can read files in the list
Returns
-------
:any:`list`: [(neg,pos)] A list of tuples, where each tuple contains the
``negative`` and ``positive`` sceach system/probee.
"""
if filenames is None:
return None
res = []
for filepath in filenames:
try:
res.append(func_load(filepath))
except:
raise
return res
def get_negatives_positives(score_lines):
"""Take the output of load_score and return negatives and positives. This
......
''' Click commands for ``bob.bio.base`` '''
import click
import bob.bio.base.script.figure as bio_figure
import bob.measure.script.figure as measure_figure
from ..score import load
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option)
def rank_option(**kwargs):
'''Get option for rank parameter'''
def custom_rank_option(func):
def callback(ctx, param, value):
value = 1 if value < 0 else value
ctx.meta['rank'] = value
return value
return click.option(
'-rk', '--rank', type=click.INT, default=1,
help='Rank for DIC',
callback=callback, show_default=True, **kwargs)(func)
return custom_rank_option
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.eval_option()
@common_options.output_log_metric_option()
@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
@common_options.cost_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kargs):
"""Prints a single output line that contains all info for a given
criterion (eer, hter, far, mindcf, cllr, rr).
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Resulting table format can be changer using the `--tablefmt`. Default
formats are `rst` when output in the terminal and `latex` when
written in a log file (see `--log`)
Examples:
$ bob bio metrics dev-scores
$ bob bio metrics --no-evaluation dev-scores1 dev-scores2
$ bob bio metrics -l results.txt dev-scores1 eval-scores1
$ bob bio metrics {dev,eval}-scores1 {dev,eval}-scores2
"""
if 'criter' in ctx.meta and ctx.meta['criter'] == 'rr':
process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc)
else:
process = bio_figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.eval_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.x_rotation_option()
@common_options.lines_at_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, evaluation, **kargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
computed using :py:func:`bob.measure.roc`.
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio roc dev-scores
$ bob bio roc dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Roc(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@common_options.lines_at_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, evaluation, **kargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio det dev-scores
$ bob bio det dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob bio det -o my_det.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Det(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(min_arg=1, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kargs):
"""Plot EPC (expected performance curve):
plots the error rate on the eval set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost
in [0; 1] of FPR and FNR when calculating the threshold.
You need to provide one or more development score and eval file(s)
for each experiment. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio epc dev-scores eval-scores
$ bob bio epc -o my_epc.pdf dev-scores1 eval-scores1
"""
process = measure_figure.Epc(ctx, scores, True, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='cmc.pdf')
@common_options.eval_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
@common_options.x_rotation_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def cmc(ctx, scores, evaluation, **kargs):
"""Plot CMC (cumulative match characteristic curve):
graphical presentation of results of an identification task eval,
plotting rank values on the x-axis and the probability of correct identification
at or below that rank on the y-axis. The values for the axis will be
computed using :py:func:`bob.measure.cmc`.
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio cmc dev-scores
$ bob bio cmc dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob bio cmc -o my_roc.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Cmc(ctx, scores, evaluation, load.cmc)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='cmc.pdf')
@common_options.eval_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
@common_options.x_rotation_option()
@rank_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def dic(ctx, scores, evaluation, **kargs):
"""Plots the Detection & Identification curve over the FAR
This curve is designed to be used in an open set identification protocol, and
defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least one
open set probe item, i.e., with no corresponding gallery, such that the
positives for that pair are ``None``.
The detection and identification curve first computes FAR thresholds based on
the out-of-set probe scores (negative scores). For each probe item, the
**maximum** negative score is used. Then, it plots the detection and
identification rates for those thresholds, which are based on the in-set
probe scores only. See [LiJain2005]_ for more details.
.. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
$ bob bio dic dev-scores
$ bob bio dic dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob bio dic -o my_roc.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Dic(ctx, scores, evaluation, load.cmc)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option()
@common_options.titles_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
computed from dev-scores. If you want to display dev-scores distributions
as well, use ``--show-dev`` option.
Examples:
$ bob bio hist dev-scores
$ bob bio hist dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob bio hist --criter --show-dev hter dev-scores1 eval-scores1
"""
process = bio_figure.Hist(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.table_option()
@common_options.eval_option()
@common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.lines_at_option()
@common_options.cost_option()
@rank_option()
@common_options.far_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, evaluation, **kwargs):
'''Evalutes score file, runs error analysis on score sets and plot curves.
\b
1. Computes the threshold using either EER, min. HTER or FAR value
criteria on development set scores
2. Applies the above threshold on eval set scores to compute the HTER, if a
eval-score set is provided
3. Computes Cllr and minCllr and minDCF
3. Reports error metrics in the console or in a log file
4. Plots ROC, EPC, DET, score distributions
curves to a multi-page PDF file
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used, the flag `--no-evaluation` must be used.
is required in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
You need to provide 2 score files for each biometric system in this order:
\b
* development scores
* evaluation scores
Examples:
$ bob bio evaluate dev-scores
$ bob bio evaluate -l metrics.txt -o my_plots.pdf dev-scores eval-scores
$ bob bio evaluate -o my_plots.pdf /path/to/syst-{1,2,3}/{dev,eval}-scores
'''
log_str = ''
if 'log' in ctx.meta and ctx.meta['log'] is not None:
log_str = ' %s' % ctx.meta['log']
# first time erase if existing file
ctx.meta['open_mode'] = 'w'
click.echo("Computing metrics with EER%s..." % log_str)
ctx.meta['criter'] = 'eer' # no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
# other times, appends the content
ctx.meta['open_mode'] = 'a'
click.echo("Computing metrics with HTER%s..." % log_str)
ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
if 'far_value' in ctx.meta and ctx.meta['far_value'] is not None:
click.echo("Computing metrics with FAR=%f%s..." %\
(ctx.meta['far_value'], log_str))
ctx.meta['criter'] = 'far' # no criterio % n passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
click.echo("Computing minDCF%s..." % log_str)
ctx.meta['criter'] = 'mindcf' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
click.echo("Computing Cllr and minCllr%s..." % log_str)
ctx.meta['criter'] = 'cllr' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
# avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False
if evaluation:
click.echo("Starting evaluate with dev and eval scores...")
else:
click.echo("Starting evaluate with dev scores only...")
click.echo("Generating ROC in %s..." % ctx.meta['output'])
ctx.forward(roc) # use class defaults plot settings
click.echo("Generating DET in %s..." % ctx.meta['output'])
ctx.forward(det) # use class defaults plot settings
if evaluation:
click.echo("Generating EPC in %s..." % ctx.meta['output'])
ctx.forward(epc) # use class defaults plot settings
# the last one closes the file
ctx.meta['closef'] = True
click.echo("Generating score histograms in %s..." % ctx.meta['output'])
ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.forward(hist)
click.echo("Evaluate successfully completed!")
......@@ -25,6 +25,7 @@ import bob.measure
from .. import score
if not os.environ.get('BOB_NO_STYLE_CHANGES'):
# make the fig size smaller so that everything becomes bigger
matplotlib.rc('figure', figsize=(4, 3))
......@@ -369,7 +370,11 @@ def main(command_line_parameters=None):
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.roc)
# create a separate figure for dev and eval
pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC for development set", args.legend_font_size, args.legend_position, args.far_line_at, min_far=args.min_far_value), bbox_inches='tight')
pdf.savefig(_plot_roc(
frrs_dev, colors, args.legends,
args.title[0] if args.title is not None else "ROC for development set",
args.legend_font_size, args.legend_position, args.far_line_at,
min_far=args.min_far_value), bbox_inches='tight')
del frrs_dev
if args.eval_files:
if args.far_line_at is not None:
......
'''Plots and measures for bob.bio.base'''
import click
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
import bob.measure
from bob.measure import plot
from tabulate import tabulate
class Roc(measure_figure.Roc):
def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
ctx.meta['x_label'] is None else ctx.meta['x_label']
self._y_label = '1 - False Non Match Rate' if 'y_label' not in \
ctx.meta or ctx.meta['y_label'] is None else ctx.meta['y_label']
class Det(measure_figure.Det):
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
ctx.meta['x_label'] is None else ctx.meta['x_label']
self._y_label = 'False Non Match Rate' if 'y_label' not in ctx.meta or\
ctx.meta['y_label'] is None else ctx.meta['y_label']
class Cmc(measure_figure.PlotBase):
''' Handles the plotting of Cmc '''
def __init__(self, ctx, scores, evaluation, func_load):
super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._title = self._title or 'CMC'
self._x_label = self._x_label or 'Rank'
self._y_label = self._y_label or 'Identification rate'
self._max_R = 0
def compute(self, idx, input_scores, input_names):
''' Plot CMC for dev and eval data using
:py:func:`bob.measure.plot.cmc`'''
mpl.figure(1)
if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc(
input_scores[0], logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('development', input_names[0], idx)
)
self._max_R = max(rank, self._max_R)
linestyle = '--'
if self._split:
mpl.figure(2)
linestyle = measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc(
input_scores[1], logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', input_names[1], idx)
)
self._max_R = max(rank, self._max_R)
else:
rank = plot.cmc(
input_scores[0], logx=self._semilogx,
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', input_names[0], idx)
)
self._max_R = max(rank, self._max_R)
class Dic(measure_figure.PlotBase):
''' Handles the plotting of DIC'''
def __init__(self, ctx, scores, evaluation, func_load):
super(Dic, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
self._title = self._title or 'DIC'
self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR'
def compute(self, idx, input_scores, input_names):
''' Plot DIC for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`'''
mpl.figure(1)
if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve(
input_scores[0], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('development', input_names[0], idx)
)
linestyle = '--'
if self._split:
mpl.figure(2)
linestyle = measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve(
input_scores[1], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', input_names[1], idx)
)
else:
plot.detection_identification_curve(
input_scores[0], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', input_names[0], idx)
)
class Metrics(measure_figure.Metrics):
''' Compute metrics from score files'''
def init_process(self):
if self._criter == 'rr':
self._thres = [None] * self.n_systems if self._thres is None else \
self._thres
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
title = self._titles[idx] if self._titles is not None else None
headers = ['' or title, 'Development %s' % input_names[0]]
if self._eval and input_scores[1] is not None:
headers.append('eval % s' % input_names[1])
if self._criter == 'rr':
rr = bob.measure.recognition_rate(input_scores[0], self._thres[idx])
dev_rr = "%.1f%%" % (100 * rr)
raws = [['RR', dev_rr]]
if self._eval and input_scores[1] is not None:
rr = bob.measure.recognition_rate(input_scores[1], self._thres[idx])