Commit ab9cd9e7 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

move click options to bob.extension, change default axis for ROC and DET to be...

move click options to bob.extension, change default axis for ROC and DET to be ISO conform, add command to generate fakes score files, add options to change title and axis labels
parent 88a4b2e4
Pipeline #19063 failed with stage
in 48 minutes and 4 seconds
......@@ -5,19 +5,19 @@ import click
from .. import load
from . import figure
from . import common_options
from bob.extension.scripts.click_helper import verbosity_option
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
@common_options.table_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kwargs):
......@@ -45,6 +45,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
......@@ -54,7 +55,9 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
......@@ -83,12 +86,15 @@ def roc(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.points_curve_option()
@common_options.const_layout_option()
@verbosity_option()
......@@ -117,10 +123,13 @@ def det(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.title_option()
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.axis_fontsize_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
......@@ -152,6 +161,7 @@ def epc(ctx, scores, **kwargs):
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option()
@common_options.title_option()
@common_options.titles_option()
@verbosity_option()
@click.pass_context
......@@ -191,7 +201,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
@common_options.fmr_line_at_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
......
......@@ -7,7 +7,8 @@ import click
from click.types import INT, FLOAT, Choice, File
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from bob.extension.scripts.click_helper import verbosity_option
from bob.extension.scripts.click_helper import (verbosity_option, bool_option,
list_float_option)
logger = logging.getLogger(__name__)
......@@ -75,30 +76,6 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
)(func)
return custom_scores_argument
def bool_option(name, short_name, desc, dflt=False, **kwargs):
'''Generic provider for boolean options
Parameters
----------
name: str
name of the option
short_name: str
short name for the option
desc: str
short description for the option
dflt: bool or None
Default value
'''
def custom_bool_option(func):
def callback(ctx, param, value):
ctx.meta[name.replace('-', '_')] = value
return value
return click.option(
'-' + short_name, '--%s/--no-%s' % (name, name), default=dflt,
help=desc,
show_default=True, callback=callback, is_eager=True, **kwargs)(func)
return custom_bool_option
def eval_option(**kwargs):
'''Get option flag to say if eval-scores are provided'''
return bool_option(
......@@ -134,47 +111,6 @@ def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
return bool_option('clayout', 'Y', '(De)Activate constrained layout', dflt)
def list_float_option(name, short_name, desc, nitems=None, dflt=None, **kwargs):
'''Get option to get a list of float f
Parameters
----------
name: str
name of the option
short_name: str
short name for the option
desc: str
short description for the option
nitems: :obj:`int`
if given, the parsed list must contains this number of items
dflt: :any:`list`
List of default values for axes.
'''
def custom_list_float_option(func):
def callback(ctx, param, value):
if value is not None:
tmp = value.split(',')
if nitems is not None and len(tmp) != nitems:
raise click.BadParameter(
'%s Must provide %d axis limits' % (name, nitems)
)
try:
value = [float(i) for i in tmp]
except:
raise click.BadParameter('Inputs of %s be floats' % name)
if None in value:
value = None
elif dflt is not None and None not in dflt and len(dflt) == nitems:
value = dflt if not all(
isinstance(x, float) for x in dflt
) else None
ctx.meta[name.replace('-', '_')] = value
return value
return click.option(
'-'+short_name, '--'+name, default=None, show_default=True,
help=desc, callback=callback, **kwargs)(func)
return custom_list_float_option
def axes_val_option(dflt=None, **kwargs):
return list_float_option(
name='axlim', short_name='L',
......@@ -189,6 +125,14 @@ def thresholds_option(**kwargs):
nitems=None, dflt=None, **kwargs
)
def lines_at_option(**kwargs):
'''Get option to draw const far line'''
return list_float_option(
name='lines-at', short_name='la',
desc='If given, draw a veritcal lines on ROC plots',
nitems=None, dflt=None, **kwargs
)
def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size'''
def custom_axis_fontsize_option(func):
......@@ -215,19 +159,6 @@ def x_rotation_option(dflt=0, **kwargs):
callback=callback, **kwargs)(func)
return custom_x_rotation_option
def fmr_line_at_option(**kwargs):
'''Get option to draw const fmr line'''
def custom_fmr_line_at_option(func):
def callback(ctx, param, value):
if value is not None:
value = min(max(0.0, value), 1.0)
ctx.meta['fmr_at'] = value
return value
return click.option(
'-L', '--fmr-at', type=float, default=None, show_default=True,
help='If given, draw a veritcal line for constant FMR on ROC plots',
callback=callback, **kwargs)(func)
return custom_fmr_line_at_option
def cost_option(**kwargs):
'''Get option to get cost for FAR'''
......@@ -343,20 +274,6 @@ def output_plot_metric_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_output_plot_file_option
def open_file_mode_option(**kwargs):
'''Get open mode file option'''
def custom_open_file_mode_option(func):
def callback(ctx, param, value):
if value not in ['w', 'a', 'w+', 'a+']:
raise click.BadParameter('Incorrect open file mode')
ctx.meta['open_mode'] = value
return value
return click.option(
'-om', '--open-mode', default='w',
help='File open mode',
callback=callback, **kwargs)(func)
return custom_open_file_mode_option
def criterion_option(lcriteria=['eer', 'hter', 'far'], **kwargs):
"""Get option flag to tell which criteriom is used (default:eer)
......@@ -494,7 +411,7 @@ def marker_style_option(**kwargs):
return custom_marker_style_option
def titles_option(**kwargs):
'''Get the titles otpion for the different systems'''
'''Get the titles option for the different systems'''
def custom_titles_option(func):
def callback(ctx, param, value):
if value is not None:
......@@ -502,12 +419,48 @@ def titles_option(**kwargs):
ctx.meta['titles'] = value
return value
return click.option(
'-t', '--titles', type=click.STRING, default=None,
'-ts', '--titles', type=click.STRING, default=None,
help='The title for each system comma separated. '
'Example: --titles ISV,CNN',
callback=callback, **kwargs)(func)
return custom_titles_option
def title_option(**kwargs):
'''Get the title option for the different systems'''
def custom_title_option(func):
def callback(ctx, param, value):
ctx.meta['title'] = value
return value
return click.option(
'-t', '--title', type=click.STRING, default=None,
help='The title of the plots',
callback=callback, **kwargs)(func)
return custom_title_option
def x_label_option(dflt=None, **kwargs):
'''Get the label option for X axis '''
def custom_x_label_option(func):
def callback(ctx, param, value):
ctx.meta['x_label'] = value
return value
return click.option(
'-xl', '--x-lable', type=click.STRING, default=dflt,
help='Label for x-axis',
callback=callback, **kwargs)(func)
return custom_x_label_option
def y_label_option(dflt=None, **kwargs):
'''Get the label option for Y axis '''
def custom_y_label_option(func):
def callback(ctx, param, value):
ctx.meta['y_label'] = value
return value
return click.option(
'-yl', '--y-lable', type=click.STRING, default=dflt,
help='Label for y-axis',
callback=callback, **kwargs)(func)
return custom_y_label_option
def style_option(**kwargs):
'''Get option for matplotlib style'''
def custom_style_option(func):
......
......@@ -403,9 +403,11 @@ class PlotBase(MeasureBase):
self._multi_plots = len(self.dev_scores) > 1
self._colors = utils.get_colors(len(self.dev_scores))
self._states = ['Development', 'Evaluation']
self._title = ''
self._x_label = 'FMR (%)'
self._y_label = 'FNMR (%)'
self._title = None if 'title' not in ctx.meta else ctx.meta['title']
self._x_label = None if 'x_label' not in ctx.meta else\
ctx.meta['x_label']
self._y_label = None if 'y_label' not in ctx.meta else\
ctx.meta['y_label']
self._grid_color = 'silver'
self._pdf_page = None
self._end_setup_plot = True
......@@ -445,7 +447,7 @@ class PlotBase(MeasureBase):
mpl.xlabel(self._x_label)
mpl.ylabel(self._y_label)
mpl.grid(True, color=self._grid_color)
mpl.legend()
mpl.legend(loc='best')
self._set_axis()
#gives warning when applied with mpl
fig.set_tight_layout(True)
......@@ -490,14 +492,18 @@ class Roc(PlotBase):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._fmr_at = None if 'fmr_at' not in ctx.meta else\
ctx.meta['fmr_at']
self._title = 'ROC'
self._x_label = 'FMR'
self._y_label = ("1 - FNMR" if self._semilogx else "FNMR")
self._far_at = None if 'lines_at' not in ctx.meta else\
ctx.meta['lines_at']
self._title = self._title or 'ROC'
self._x_label = self._x_label or 'False Positive Rate'
self._y_label = self._y_label or (
"1 - False Negative Rate" if self._semilogx else "False Negative Rate"
)
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
if self._far_at is not None:
self._eval_points = {line: [] for line in self._far_at}
def compute(self, idx, dev_score, dev_file=None,
eval_score=None, eval_file=None):
......@@ -523,12 +529,14 @@ class Roc(PlotBase):
color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', eval_file, idx, **self._kwargs)
)
if self._fmr_at is not None:
if self._far_at is not None:
from .. import farfrr
eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, self._fmr_at)
if self._semilogx:
eval_fnmr = 1 - eval_fnmr
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
for line in self._far_at:
eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, line)
if self._semilogx:
eval_fnmr = 1 - eval_fnmr
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
self._eval_points[line].append((eval_fmr, eval_fnmr))
else:
plot.roc(
dev_neg, dev_pos, self._points, self._semilogx,
......@@ -540,16 +548,31 @@ class Roc(PlotBase):
''' Draw vertical line on the dev plot at the given fmr and print the
corresponding points on the eval plot for all the systems '''
#draw vertical lines
if self._fmr_at is not None:
mpl.figure(1)
mpl.plot([self._fmr_at, self._fmr_at], [0., 1.], "--", color='black')
if self._far_at is not None:
for line in self._far_at:
mpl.figure(1)
mpl.plot([line, line], [0., 1.], "--", color='black')
if self._eval and self._split:
mpl.figure(2)
x_values = [i for i, _ in self._eval_points[line]]
y_values = [j for _, j in self._eval_points[line]]
sort_indice = sorted(
range(len(x_values)), key=x_values.__getitem__
)
x_values = [x_values[i] for i in sort_indice]
y_values = [y_values[i] for i in sort_indice]
mpl.plot(x_values,
y_values, '--',
color='black')
super(Roc, self).end_process()
class Det(PlotBase):
''' Handles the plotting of DET '''
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._title = 'DET'
self._title = self._title or 'DET'
self._x_label = self._x_label or 'False Positive Rate'
self._y_label = self._y_label or 'False Negative Rate'
#custom defaults here
if self._x_rotation is None:
self._x_rotation = 50
......@@ -595,9 +618,9 @@ class Epc(PlotBase):
super(Epc, self).__init__(ctx, scores, evaluation, func_load)
if 'eval_scores_0' not in self._ctx.meta:
raise click.UsageError("EPC requires dev and eval score files")
self._title = 'EPC'
self._x_label = 'Cost'
self._y_label = 'Min. HTER (%)'
self._title = self._title or 'EPC'
self._x_label = self._x_label or 'Cost'
self._y_label = self._y_label or 'Min. HTER (%)'
self._eval = True #always eval data with EPC
self._split = False
self._nb_figs = 1
......@@ -645,9 +668,9 @@ class Hist(PlotBase):
)
self._criter = None if 'criter' not in ctx.meta else ctx.meta['criter']
self._y_label = 'Dev. probability density' if self._eval else \
'density'
'density' or self._y_label
self._x_label = 'Scores' if not self._eval else ''
self._title_base = 'Scores'
self._title_base = self._title or 'Scores'
self._end_setup_plot = False
def compute(self, idx, dev_score, dev_file=None,
......
"""Generate random scores.
"""
import pkg_resources # to make sure bob gets imported properly
import os
import logging
import numpy
import click
from click.types import FLOAT
from bob.extension.scripts.click_helper import verbosity_option
from bob.core import random
from bob.io.base import create_directories_safe
logger = logging.getLogger(__name__)
NUM_NEG = 5000
NUM_POS = 5000
def gen_score_distr(mean_neg, mean_pos, sigma_neg=1, sigma_pos=1):
"""Generate scores from normal distributions
Parameters
----------
mean_neg : float
Mean for negative scores
mean_pos : float
Mean for positive scores
sigma_neg : float
STDev for negative scores
sigma_pos : float
STDev for positive scores
Returns
-------
neg_scores : array_like
Negatives scores
pos_scores : array_like
Positive scores
"""
mt = random.mt19937() # initialise the random number generator
neg_generator = random.normal(numpy.float32, mean_neg, sigma_neg)
pos_generator = random.normal(numpy.float32, mean_pos, sigma_pos)
neg_scores = [neg_generator(mt) for _ in range(NUM_NEG)]
pos_scores = [pos_generator(mt) for _ in range(NUM_NEG)]
return neg_scores, pos_scores
def write_scores_to_file(pos, neg, filename):
"""Writes score distributions into 2-column score files. For the format of
the 2-column score files, please refer to Bob's documentation. See
:py:func:`bob.measure.load.split`.
Parameters
----------
pos : array_like
Scores for positive samples.
neg : array_like
Scores for negative samples.
filename : str
The path to write the score to.
"""
create_directories_safe(os.path.dirname(filename))
mt = random.mt19937()
nan_dist = random.uniform(numpy.float32, 0, 1)
with open(filename, 'wt') as f:
for i in pos:
text = '1 %f\n' % i if nan_dist(mt) > 0.01 else '1 nan\n'
f.write(text)
for i in neg:
text = '-1 %f\n' % i if nan_dist(mt) > 0.01 else '1 nan\n'
f.write(text)
@click.command()
@click.argument('outdir')
@click.option('--mean-neg', default=-1, type=FLOAT, show_default=True)
@click.option('--mean-pos', default=1, type=FLOAT, show_default=True)
@verbosity_option()
def gen(outdir, mean_neg, mean_pos):
"""Generate random scores.
Generates random scores for negative and positive scores, whatever they
could be. The
scores are generated using Gaussian distribution whose mean is an input
parameter. The generated scores can be used as hypothetical datasets.
"""
# Generate the data
neg_dev, pos_dev = gen_score_distr(mean_neg, mean_pos)
neg_eval, pos_eval = gen_score_distr(mean_neg, mean_pos)
# Write the data into files
write_scores_to_file(neg_dev, pos_dev,
os.path.join(outdir, 'scores-dev'))
write_scores_to_file(neg_eval, pos_eval,
os.path.join(outdir, 'scores-eval'))
......@@ -15,7 +15,7 @@ def test_metrics():
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
......@@ -30,15 +30,16 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-t',
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-ts',
'A,B']
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--no-evaluation', dev1, dev2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
......@@ -58,7 +59,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
......@@ -66,7 +67,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_det():
......@@ -86,14 +87,15 @@ def test_det():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
......@@ -103,7 +105,8 @@ def test_epc():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
......@@ -112,7 +115,7 @@ def test_epc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
......@@ -124,7 +127,7 @@ def test_hist():
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--no-evaluation', '--criter', 'hter',
......@@ -132,7 +135,7 @@ def test_hist():
30, dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer','--output',
......@@ -140,7 +143,7 @@ def test_hist():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate():
......@@ -151,15 +154,15 @@ def test_evaluate():
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['--no-evaluation', dev1])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, ['--no-evaluation', '--output', 'my_plots.pdf', '-b',
30, '-n', 300, dev1, dev2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, [dev1, test1, dev2, test2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -78,6 +78,7 @@ setup(
'det = bob.measure.script.commands:det',
'epc = bob.measure.script.commands:epc',
'hist = bob.measure.script.commands:hist',
'gen = bob.measure.script.gen:gen',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment