Commit 0af00f10 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'title' into 'master'

Title and histograms subplots

See merge request !61
parents 40a7e636 db43e6a0
Pipeline #19892 passed with stages
in 36 minutes and 41 seconds
......@@ -10,6 +10,7 @@ import numpy
LOGGER = logging.getLogger('bob.measure')
def split(filename):
"""split(filename) -> negatives, positives
......@@ -36,7 +37,7 @@ def split(filename):
columns = numpy.loadtxt(filename)
neg_pos = columns[:, 0]
scores = columns[:, 1]
except:
except Exception:
LOGGER.error('''Cannot read {}. This file must be a two columns file with
the first column containing -1 or 1 (i.e. negative or
positive) and the second the scores
......@@ -45,6 +46,7 @@ def split(filename):
return (scores[numpy.where(neg_pos == -1)],
scores[numpy.where(neg_pos == 1)])
def split_files(filenames):
"""split_files
......
......@@ -8,6 +8,7 @@ from . import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
......@@ -22,11 +23,11 @@ from bob.extension.scripts.click_helper import (verbosity_option,
@click.pass_context
def metrics(ctx, scores, evaluation, **kwargs):
"""Prints a table that contains FtA, FAR, FRR, FMR, FMNR, HTER for a given
threshold criterion (eer or hter).
threshold criterion (eer or min-hter).
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
Resulting table format can be changed using the `--tablefmt`.
......@@ -40,15 +41,18 @@ def metrics(ctx, scores, evaluation, **kwargs):
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.title_option()
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='lower-right')
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.eval_option()
@common_options.points_curve_option()
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axes_val_option(dflt='1e-4,1,1e-4,1')
@common_options.min_far_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
......@@ -66,29 +70,32 @@ def roc(ctx, scores, evaluation, **kwargs):
false non match rate on the vertical axis. The values for the axis will be
computed using :py:func:`bob.measure.roc`.
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
Examples:
$ bob measure roc dev-scores
$ bob measure roc -v dev-scores
$ bob measure roc dev-scores1 eval-scores1 dev-scores2
$ bob measure roc -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
$ bob measure roc -v -o my_roc.pdf dev-scores1 eval-scores1
"""
process = figure.Roc(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.title_option()
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.axes_val_option(dflt='0.01,95,0.01,95')
@common_options.min_far_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
......@@ -106,26 +113,29 @@ def det(ctx, scores, evaluation, **kwargs):
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
Examples:
$ bob measure det dev-scores
$ bob measure det -v dev-scores
$ bob measure det dev-scores1 eval-scores1 dev-scores2
$ bob measure det -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure det -o my_det.pdf dev-scores1 eval-scores1
$ bob measure det -v -o my_det.pdf dev-scores1 eval-scores1
"""
process = figure.Det(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(min_arg=1, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.title_option()
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-center')
@common_options.points_curve_option()
@common_options.const_layout_option()
@common_options.x_label_option()
......@@ -145,13 +155,14 @@ def epc(ctx, scores, **kwargs):
for each experiment.
Examples:
$ bob measure epc dev-scores eval-scores
$ bob measure epc -v dev-scores eval-scores
$ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
$ bob measure epc -v -o my_epc.pdf dev-scores1 eval-scores1
"""
process = figure.Epc(ctx, scores, True, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
......@@ -160,22 +171,24 @@ def epc(ctx, scores, **kwargs):
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option()
@common_options.title_option()
@common_options.legends_option()
@common_options.figsize_option()
@common_options.figsize_option(dflt=None)
@common_options.style_option()
@common_options.linestyles_option()
@common_options.subplot_option()
@common_options.legend_ncols_option()
@common_options.no_legend_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
You need to provide one or more development score file(s) for each experiment.
You can also provide evaluation files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
......@@ -183,16 +196,17 @@ def hist(ctx, scores, evaluation, **kwargs):
as well, use ``--show-dev`` option.
Examples:
$ bob measure hist dev-scores
$ bob measure hist -v dev-scores
$ bob measure hist dev-scores1 eval-scores1 dev-scores2
$ bob measure hist -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure hist --criterion hter --show-dev dev-scores1 eval-scores1
$ bob measure hist -v --criterion min-hter --show-dev dev-scores1 eval-scores1
"""
process = figure.Hist(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.legends_option()
......@@ -202,7 +216,6 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.n_bins_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@common_options.figsize_option()
......@@ -216,8 +229,8 @@ def evaluate(ctx, scores, evaluation, **kwargs):
\b
1. Computes the threshold using either EER or min. HTER criteria on
development set scores
2. Applies the above threshold on evaluation set scores to compute the HTER, if a
eval-score set is provided
2. Applies the above threshold on evaluation set scores to compute the
HTER, if a eval-score set is provided
3. Reports error rates on the console
4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
file
......@@ -230,14 +243,14 @@ def evaluate(ctx, scores, evaluation, **kwargs):
* evaluation scores
Examples:
$ bob measure evaluate dev-scores
$ bob measure evaluate -v dev-scores
$ bob measure evaluate scores-dev1 scores-eval1 scores-dev2
$ bob measure evaluate -v scores-dev1 scores-eval1 scores-dev2
scores-eval2
$ bob measure evaluate /path/to/sys-{1,2,3}/scores-{dev,eval}
$ bob measure evaluate -v /path/to/sys-{1,2,3}/scores-{dev,eval}
$ bob measure evaluate -l metrics.txt -o my_plots.pdf dev-scores eval-scores
$ bob measure evaluate -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
'''
# first time erase if existing file
ctx.meta['open_mode'] = 'w'
......@@ -246,8 +259,8 @@ def evaluate(ctx, scores, evaluation, **kwargs):
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
# second time, appends the content
ctx.meta['open_mode'] = 'a'
click.echo("Computing metrics with HTER...")
ctx.meta['criterion'] = 'hter' # no criterion passed in evaluate
click.echo("Computing metrics with min-HTER...")
ctx.meta['criterion'] = 'min-hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log'])
......@@ -260,12 +273,12 @@ def evaluate(ctx, scores, evaluation, **kwargs):
click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...")
# set axes limits for ROC
ctx.forward(roc) # use class defaults plot settings
ctx.forward(roc) # use class defaults plot settings
click.echo("Computing DET...")
ctx.forward(det) # use class defaults plot settings
ctx.forward(det) # use class defaults plot settings
if evaluation:
click.echo("Computing EPC...")
ctx.forward(epc) # use class defaults plot settings
ctx.forward(epc) # use class defaults plot settings
# the last one closes the file
ctx.meta['closef'] = True
click.echo("Computing score histograms...")
......
......@@ -4,11 +4,13 @@ import logging
import click
from click.types import INT, FLOAT
import matplotlib.pyplot as plt
import tabulate
from matplotlib.backends.backend_pdf import PdfPages
from bob.extension.scripts.click_helper import (bool_option, list_float_option)
LOGGER = logging.getLogger(__name__)
def scores_argument(min_arg=1, force_eval=False, **kwargs):
"""Get the argument for scores, and add `dev-scores` and `eval-scores` in
the context when `--evaluation` flag is on (default)
......@@ -35,7 +37,7 @@ def scores_argument(min_arg=1, force_eval=False, **kwargs):
if 'train' in ctx.meta and ctx.meta['train']:
mutli += 1
error += '- %d training file(s) \n' % min_a
#add more test here if other inputs are needed
# add more test here if other inputs are needed
min_a *= mutli
ctx.meta['min_arg'] = min_a
......@@ -54,6 +56,15 @@ def scores_argument(min_arg=1, force_eval=False, **kwargs):
)(func)
return custom_scores_argument
def no_legend_option(dflt=True, **kwargs):
'''Get option flag to say if legend should be displayed or not'''
return bool_option(
'disp-legend', 'dl', 'If set, no legend will be printed.',
dflt=dflt
)
def eval_option(**kwargs):
'''Get option flag to say if eval-scores are provided'''
return bool_option(
......@@ -61,39 +72,40 @@ def eval_option(**kwargs):
dflt=True
)
def sep_dev_eval_option(dflt=True, **kwargs):
'''Get option flag to say if dev and eval plots should be in different
plots'''
return bool_option(
'split', 's','If set, evaluation and dev curve in different plots',
'split', 's', 'If set, evaluation and dev curve in different plots',
dflt
)
def linestyles_option(dflt=False, **kwargs):
''' Get option flag to turn on/off linestyles'''
return bool_option('line-linestyles', 'S', 'If given, applies a different '
'linestyles to each line.', dflt, **kwargs)
def cmc_option(**kwargs):
'''Get option flag to say if cmc scores'''
return bool_option('cmc', 'C', 'If set, CMC score files are provided',
**kwargs)
def semilogx_option(dflt=False, **kwargs):
'''Option to use semilog X-axis'''
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt,
**kwargs)
def show_dev_option(dflt=False, **kwargs):
'''Option to tell if should show dev histo'''
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt,
**kwargs)
def print_filenames_option(dflt=True, **kwargs):
'''Option to tell if filenames should be in the title'''
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt,
**kwargs)
def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
def custom_layout_option(func):
......@@ -107,6 +119,7 @@ def const_layout_option(dflt=True, **kwargs):
callback=callback, **kwargs)(func)
return custom_layout_option
def axes_val_option(dflt=None, **kwargs):
''' Option for setting min/max values on axes '''
return list_float_option(
......@@ -116,6 +129,7 @@ def axes_val_option(dflt=None, **kwargs):
nitems=4, dflt=dflt, **kwargs
)
def thresholds_option(**kwargs):
''' Option to give a list of thresholds '''
return list_float_option(
......@@ -125,14 +139,17 @@ def thresholds_option(**kwargs):
nitems=None, dflt=None, **kwargs
)
def lines_at_option(**kwargs):
def lines_at_option(dflt='1e-3', **kwargs):
'''Get option to draw const far line'''
return list_float_option(
name='lines-at', short_name='la',
desc='If given, draw veritcal lines at the given axis positions',
nitems=None, dflt=None, **kwargs
desc='If given, draw vertical lines at the given axis positions. '
'Your values must be separated with a comma (,) without space.',
nitems=None, dflt=dflt, **kwargs
)
def x_rotation_option(dflt=0, **kwargs):
'''Get option for rotartion of the x axis lables'''
def custom_x_rotation_option(func):
......@@ -141,11 +158,44 @@ def x_rotation_option(dflt=0, **kwargs):
ctx.meta['x_rotation'] = value
return value
return click.option(
'-r', '--x-rotation', type=click.INT, default=dflt, show_default=True,
help='X axis labels ration',
'-r', '--x-rotation', type=click.INT, default=dflt,
show_default=True, help='X axis labels ration',
callback=callback, **kwargs)(func)
return custom_x_rotation_option
def legend_ncols_option(dflt=10, **kwargs):
'''Get option for number of columns for legends'''
def custom_legend_ncols_option(func):
def callback(ctx, param, value):
value = abs(value)
ctx.meta['legends_ncol'] = value
return value
return click.option(
'-lc', '--legends-ncol', type=click.INT, default=dflt,
show_default=True,
help='The number of columns of the legend layout.',
callback=callback, **kwargs)(func)
return custom_legend_ncols_option
def subplot_option(dflt=111, **kwargs):
'''Get option to set subplots'''
def custom_subplot_option(func):
def callback(ctx, param, value):
value = abs(value)
nrows = value // 10
nrows, ncols = divmod(nrows, 10)
ctx.meta['n_col'] = ncols
ctx.meta['n_row'] = nrows
return value
return click.option(
'-sp', '--subplot', type=click.INT, default=dflt,
show_default=True, help='The order of subplots.',
callback=callback, **kwargs)(func)
return custom_subplot_option
def cost_option(**kwargs):
'''Get option to get cost for FAR'''
def custom_cost_option(func):
......@@ -160,14 +210,15 @@ def cost_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_cost_option
def points_curve_option(**kwargs):
'''Get the number of points use to draw curves'''
def custom_points_curve_option(func):
def callback(ctx, param, value):
if value < 2:
raise click.BadParameter(
'Number of points to draw curves must be greater than 1'
, ctx=ctx
'Number of points to draw curves must be greater than 1',
ctx=ctx
)
ctx.meta['points'] = value
return value
......@@ -177,6 +228,7 @@ def points_curve_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_points_curve_option
def n_bins_option(**kwargs):
'''Get the number of bins in the histograms'''
def custom_n_bins_option(func):
......@@ -199,24 +251,22 @@ def n_bins_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_n_bins_option
def table_option(**kwargs):
'''Get table option for tabulate package
More informnations: https://pypi.python.org/pypi/tabulate
More informnations: https://pypi.org/project/tabulate/
'''
def custom_table_option(func):
def callback(ctx, param, value):
ctx.meta['tablefmt'] = value
return value
return click.option(
'--tablefmt', type=click.STRING, default='rst',
show_default=True, help='Format for table display: `plain`, '
'`simple`, `grid`, `fancy_grid`, `pipe`, `orgtbl`, '
'`jira`, `presto`, `psql`, `rst`, `mediawiki`, `moinmoin`, '
'`youtrack`, `html`, `latex`, '
'`latex_raw`, `latex_booktabs`, `textile`',
callback=callback,**kwargs)(func)
'--tablefmt', type=click.Choice(tabulate.tabulate_formats),
default='rst', show_default=True, help='Format of printed tables.',
callback=callback, **kwargs)(func)
return custom_table_option
def output_plot_file_option(default_out='plots.pdf', **kwargs):
'''Get options for output file for plots'''
def custom_output_plot_file_option(func):
......@@ -234,6 +284,7 @@ def output_plot_file_option(default_out='plots.pdf', **kwargs):
callback=callback, **kwargs)(func)
return custom_output_plot_file_option
def output_log_metric_option(**kwargs):
'''Get options for output file for metrics'''
def custom_output_log_file_option(func):
......@@ -245,11 +296,12 @@ def output_log_metric_option(**kwargs):
return click.option(
'-l', '--log', default=None, type=click.STRING,
help='If provided, computed numbers are written to '
'this file instead of the standard output.',
'this file instead of the standard output.',
callback=callback, **kwargs)(func)
return custom_output_log_file_option
def criterion_option(lcriteria=['eer', 'hter', 'far'], **kwargs):
def criterion_option(lcriteria=['eer', 'min-hter', 'far'], **kwargs):
"""Get option flag to tell which criteriom is used (default:eer)
Parameters
......@@ -260,7 +312,7 @@ def criterion_option(lcriteria=['eer', 'hter', 'far'], **kwargs):
def custom_criterion_option(func):
def callback(ctx, param, value):
list_accepted_crit = lcriteria if lcriteria is not None else \
['eer', 'hter', 'far']
['eer', 'min-hter', 'far']
if value not in list_accepted_crit:
raise click.BadParameter('Incorrect value for `--criterion`. '
'Must be one of [`%s`]' %
......@@ -268,11 +320,13 @@ def criterion_option(lcriteria=['eer', 'hter', 'far'], **kwargs):
ctx.meta['criterion'] = value
return value
return click.option(
'--criterion', default='eer', help='Criterion to compute plots and '
'metrics: `eer` (default), `hter`',
callback=callback, is_eager=True ,**kwargs)(func)
'-c', '--criterion', default='eer',
help='Criterion to compute plots and '
'metrics: `eer`, `min-hter` or `far`',
callback=callback, is_eager=True, **kwargs)(func)
return custom_criterion_option
def far_option(**kwargs):
'''Get option to get far value'''
def custom_far_option(func):
......@@ -284,9 +338,10 @@ def far_option(**kwargs):
return click.option(
'-f', '--far-value', type=click.FLOAT, default=None,
help='The FAR value for which to compute metrics',
callback=callback, show_default=True,**kwargs)(func)
callback=callback, show_default=True, **kwargs)(func)
return custom_far_option
def min_far_option(dflt=1e-4, **kwargs):
'''Get option to get min far value'''
def custom_min_far_option(func):
......@@ -299,9 +354,10 @@ def min_far_option(dflt=1e-4, **kwargs):
'-M', '--min-far-value', type=click.FLOAT, default=dflt,
help='Select the minimum FAR value used in ROC and DET plots; '
'should be a power of 10.',
callback=callback, show_default=True,**kwargs)(func)
callback=callback, show_default=True, **kwargs)(func)
return custom_min_far_option
def figsize_option(dflt='4,3', **kwargs):
"""Get option for matplotlib figsize
......@@ -324,35 +380,31 @@ def figsize_option(dflt='4,3', **kwargs):
plt.rcParams['figure.figsize'] = ctx.meta['figsize']
return value
return click.option(
'--figsize', default=dflt, help='If given, will run '
'``plt.rcParams[\'figure.figsize\']=figsize)``. Example: --fig-size 4,6',
'--figsize', default=dflt, show_default=True,
help='If given, will run '
'``plt.rcParams[\'figure.figsize\']=figsize)``. '
'Example: --fig-size 4,6',
callback=callback, **kwargs)(func)
return custom_figsize_option
def legend_ncols_option(**kwargs):
'''Get the number of columns to set in the legend of the plot'''
def custom_legend_ncols_option(func):
def callback(ctx, param, value):
ctx.meta['legend_ncol'] = value
return value
return click.option(
'--legend-ncol', default=3, show_default=True,
type=INT, help='The number of columns of the legend layout.',
callback=callback, **kwargs)(func)
return custom_legend_ncols_option
def legend_loc_option(**kwargs):
'''Get tthe legend location of the plot'''
def legend_loc_option(dflt='best', **kwargs):
'''Get the legend location of the plot'''
def custom_legend_loc_option(func):
def callback(ctx, param, value):
ctx.meta['legend_loc'] = value
ctx.meta['legend_loc'] = value.replace('-', ' ')
return value
return click.option(
'--legend-location', default=0, show_default=True,
type=INT, help='The lengend location code',
'-lc', '--legend-loc', default=dflt, show_default=True,
type=click.Choice(['best', 'upper-right', 'upper-left',
'lower-left', 'lower-right', 'right',
'center-left', 'center-right', 'lower-center',
'upper-center', 'center']),
help='The legend location code',
callback=callback, **kwargs)(func)
return custom_legend_loc_option
def line_width_option(**kwargs):
'''Get line width option for the plots'''
def custom_line_width_option(func):
......@@ -365,6 +417,7 @@ def line_width_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_line_width_option
def marker_style_option(**kwargs):