Commit 00a68cf5 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Detailed option description, matplolib setting moved to callback functions,...

Detailed option description, matplolib setting moved to callback functions, add vertical line option to DET
parent 1dbb8817
Pipeline #19159 passed with stage
in 33 minutes and 40 seconds
...@@ -142,7 +142,7 @@ def roc_for_far(negatives, positives, far_values=log_values(), **kwargs): ...@@ -142,7 +142,7 @@ def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
from matplotlib import pyplot from matplotlib import pyplot
from . import roc_for_far as calc from . import roc_for_far as calc
out = calc(negatives, positives, far_values) out = calc(negatives, positives, far_values)
return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs) return pyplot.semilogx(out[0, :], (1 - out[1, :]), **kwargs)
def precision_recall_curve(negatives, positives, npoints=100, **kwargs): def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
......
...@@ -48,7 +48,6 @@ def metrics(ctx, scores, evaluation, **kwargs): ...@@ -48,7 +48,6 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.output_plot_file_option(default_out='roc.pdf') @common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.eval_option() @common_options.eval_option()
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1]) @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.x_rotation_option() @common_options.x_rotation_option()
@common_options.x_label_option() @common_options.x_label_option()
...@@ -92,6 +91,7 @@ def roc(ctx, scores, evaluation, **kwargs): ...@@ -92,6 +91,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.x_label_option() @common_options.x_label_option()
@common_options.y_label_option() @common_options.y_label_option()
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.lines_at_option()
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
...@@ -196,7 +196,6 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -196,7 +196,6 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.output_plot_metric_option() @common_options.output_plot_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf') @common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option() @common_options.n_bins_option()
@common_options.lines_at_option() @common_options.lines_at_option()
@common_options.const_layout_option() @common_options.const_layout_option()
......
...@@ -106,13 +106,23 @@ def print_filenames_option(dflt=True, **kwargs): ...@@ -106,13 +106,23 @@ def print_filenames_option(dflt=True, **kwargs):
def const_layout_option(dflt=True, **kwargs): def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout''' '''Option to set matplotlib constrained_layout'''
return bool_option('clayout', 'Y', '(De)Activate constrained layout', dflt) def custom_layout_option(func):
def callback(ctx, param, value):
ctx.meta['clayout'] = value
plt.rcParams['figure.constrained_layout.use'] = value
return value
return click.option(
'-Y', '--clayout/--no-clayout', default=dflt, show_default=True,
help='(De)Activate constrained layout',
callback=callback, **kwargs)(func)
return custom_layout_option
def axes_val_option(dflt=None, **kwargs): def axes_val_option(dflt=None, **kwargs):
''' Option for setting min/max values on axes ''' ''' Option for setting min/max values on axes '''
return list_float_option( return list_float_option(
name='axlim', short_name='L', name='axlim', short_name='L',
desc='min/max axes values separated by commas (min_x, max_x, min_y, max_y)', desc='min/max axes values separated by commas (e.g. ``--axlim '
' 0.1,100,0.1,100``)',
nitems=4, dflt=dflt, **kwargs nitems=4, dflt=dflt, **kwargs
) )
...@@ -120,7 +130,8 @@ def thresholds_option(**kwargs): ...@@ -120,7 +130,8 @@ def thresholds_option(**kwargs):
''' Option to give a list of thresholds ''' ''' Option to give a list of thresholds '''
return list_float_option( return list_float_option(
name='thres', short_name='T', name='thres', short_name='T',
desc='Given threshold for metrics computations', desc='Given threshold for metrics computations, e.g. '
'0.005,0.001,0.056',
nitems=None, dflt=None, **kwargs nitems=None, dflt=None, **kwargs
) )
...@@ -128,7 +139,7 @@ def lines_at_option(**kwargs): ...@@ -128,7 +139,7 @@ def lines_at_option(**kwargs):
'''Get option to draw const far line''' '''Get option to draw const far line'''
return list_float_option( return list_float_option(
name='lines-at', short_name='la', name='lines-at', short_name='la',
desc='If given, draw veritcal lines on ROC plots', desc='If given, draw veritcal lines at the given axis positions',
nitems=None, dflt=None, **kwargs nitems=None, dflt=None, **kwargs
) )
...@@ -291,11 +302,12 @@ def figsize_option(**kwargs): ...@@ -291,11 +302,12 @@ def figsize_option(**kwargs):
def callback(ctx, param, value): def callback(ctx, param, value):
ctx.meta['figsize'] = value if value is None else \ ctx.meta['figsize'] = value if value is None else \
[float(x) for x in value.split(',')] [float(x) for x in value.split(',')]
plt.figure(figsize=ctx.meta['figsize']) if value is not None:
plt.rcParams['figure.figsize'] = ctx.meta['figsize']
return value return value
return click.option( return click.option(
'--figsize', help='If given, will run ' '--figsize', help='If given, will run '
'``plt.figure(figsize=figsize)``. Example: --fig-size 4,6', '``plt.rcParams[\'figure.figsize\']=figsize)``. Example: --fig-size 4,6',
callback=callback, **kwargs)(func) callback=callback, **kwargs)(func)
return custom_figsize_option return custom_figsize_option
......
...@@ -9,8 +9,7 @@ import matplotlib ...@@ -9,8 +9,7 @@ import matplotlib
import matplotlib.pyplot as mpl import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate from tabulate import tabulate
from .. import plot from .. import (far_threshold, plot, utils, ppndf)
from .. import utils
LINESTYLES = [ LINESTYLES = [
(0, ()), #solid (0, ()), #solid
...@@ -329,6 +328,12 @@ class PlotBase(MeasureBase): ...@@ -329,6 +328,12 @@ class PlotBase(MeasureBase):
self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim'] self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._clayout = None if 'clayout' not in ctx.meta else\ self._clayout = None if 'clayout' not in ctx.meta else\
ctx.meta['clayout'] ctx.meta['clayout']
self._far_at = None if 'lines_at' not in ctx.meta else\
ctx.meta['lines_at']
self._trans_far_val = self._far_at
if self._far_at is not None:
self._eval_points = {line: [] for line in self._far_at}
self._lines_val = []
self._print_fn = True if 'show_fn' not in ctx.meta else\ self._print_fn = True if 'show_fn' not in ctx.meta else\
ctx.meta['show_fn'] ctx.meta['show_fn']
self._x_rotation = None if 'x_rotation' not in ctx.meta else \ self._x_rotation = None if 'x_rotation' not in ctx.meta else \
...@@ -365,8 +370,28 @@ class PlotBase(MeasureBase): ...@@ -365,8 +370,28 @@ class PlotBase(MeasureBase):
fig.clear() fig.clear()
def end_process(self): def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and ''' Set title, legend, axis labels, grid colors, save figures, drow
close pdf is needed ''' lines and close pdf if needed '''
#draw vertical lines
if self._far_at is not None:
for (line, line_trans) in zip(self._far_at, self._trans_far_val):
mpl.figure(1)
mpl.plot(
[line_trans, line_trans], [-100.0, 100.], "--",
color='black'
)
if self._eval and self._split:
mpl.figure(2)
x_values = [i for i, _ in self._eval_points[line]]
y_values = [j for _, j in self._eval_points[line]]
sort_indice = sorted(
range(len(x_values)), key=x_values.__getitem__
)
x_values = [x_values[i] for i in sort_indice]
y_values = [y_values[i] for i in sort_indice]
mpl.plot(x_values,
y_values, '--',
color='black')
#only for plots #only for plots
if self._end_setup_plot: if self._end_setup_plot:
for i in range(self._nb_figs): for i in range(self._nb_figs):
...@@ -390,7 +415,6 @@ class PlotBase(MeasureBase): ...@@ -390,7 +415,6 @@ class PlotBase(MeasureBase):
('closef' not in self._ctx.meta or self._ctx.meta['closef']): ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
self._pdf_page.close() self._pdf_page.close()
#common protected functions #common protected functions
def _label(self, base, name, idx): def _label(self, base, name, idx):
...@@ -408,20 +432,12 @@ class Roc(PlotBase): ...@@ -408,20 +432,12 @@ class Roc(PlotBase):
''' Handles the plotting of ROC''' ''' Handles the plotting of ROC'''
def __init__(self, ctx, scores, evaluation, func_load): def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load) super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._far_at = None if 'lines_at' not in ctx.meta else\
ctx.meta['lines_at']
self._title = self._title or 'ROC' self._title = self._title or 'ROC'
self._x_label = self._x_label or 'False Positive Rate' self._x_label = self._x_label or 'False Positive Rate'
self._y_label = self._y_label or ( self._y_label = self._y_label or "1 - False Negative Rate"
"1 - False Negative Rate" if self._semilogx else "False Negative Rate"
)
#custom defaults #custom defaults
if self._axlim is None: if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0] self._axlim = [1e-4, 1.0, 1e-4, 1.0]
if self._far_at is not None:
self._eval_points = {line: [] for line in self._far_at}
def compute(self, idx, dev_score, dev_file=None, def compute(self, idx, dev_score, dev_file=None,
eval_score=None, eval_file=None): eval_score=None, eval_file=None):
...@@ -432,8 +448,8 @@ class Roc(PlotBase): ...@@ -432,8 +448,8 @@ class Roc(PlotBase):
mpl.figure(1) mpl.figure(1)
if self._eval: if self._eval:
linestyle = '-' if not self._split else LINESTYLES[idx % 14] linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.roc( plot.roc_for_far(
dev_neg, dev_pos, self._points, self._semilogx, dev_neg, dev_pos,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('development', dev_file, idx, **self._kwargs) label=self._label('development', dev_file, idx, **self._kwargs)
) )
...@@ -442,48 +458,26 @@ class Roc(PlotBase): ...@@ -442,48 +458,26 @@ class Roc(PlotBase):
mpl.figure(2) mpl.figure(2)
linestyle = LINESTYLES[idx % 14] linestyle = LINESTYLES[idx % 14]
plot.roc( plot.roc_for_far(
eval_neg, eval_pos, self._points, self._semilogx, eval_neg, eval_pos,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', eval_file, idx, **self._kwargs) label=self._label('eval', eval_file, idx, **self._kwargs)
) )
if self._far_at is not None: if self._far_at is not None:
from .. import farfrr from .. import farfrr
for line in self._far_at: for line in self._far_at:
eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, line) thres_line = far_threshold(dev_neg, dev_pos, line)
if self._semilogx: eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, thres_line)
eval_fnmr = 1 - eval_fnmr eval_fnmr = 1 - eval_fnmr
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30) mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
self._eval_points[line].append((eval_fmr, eval_fnmr)) self._eval_points[line].append((eval_fmr, eval_fnmr))
else: else:
plot.roc( plot.roc_for_far(
dev_neg, dev_pos, self._points, self._semilogx, dev_neg, dev_pos,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14], color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
label=self._label('development', dev_file, idx, **self._kwargs) label=self._label('development', dev_file, idx, **self._kwargs)
) )
def end_process(self):
''' Draw vertical line on the dev plot at the given fmr and print the
corresponding points on the eval plot for all the systems '''
#draw vertical lines
if self._far_at is not None:
for line in self._far_at:
mpl.figure(1)
mpl.plot([line, line], [0., 1.], "--", color='black')
if self._eval and self._split:
mpl.figure(2)
x_values = [i for i, _ in self._eval_points[line]]
y_values = [j for _, j in self._eval_points[line]]
sort_indice = sorted(
range(len(x_values)), key=x_values.__getitem__
)
x_values = [x_values[i] for i in sort_indice]
y_values = [y_values[i] for i in sort_indice]
mpl.plot(x_values,
y_values, '--',
color='black')
super(Roc, self).end_process()
class Det(PlotBase): class Det(PlotBase):
''' Handles the plotting of DET ''' ''' Handles the plotting of DET '''
def __init__(self, ctx, scores, evaluation, func_load): def __init__(self, ctx, scores, evaluation, func_load):
...@@ -491,6 +485,8 @@ class Det(PlotBase): ...@@ -491,6 +485,8 @@ class Det(PlotBase):
self._title = self._title or 'DET' self._title = self._title or 'DET'
self._x_label = self._x_label or 'False Positive Rate' self._x_label = self._x_label or 'False Positive Rate'
self._y_label = self._y_label or 'False Negative Rate' self._y_label = self._y_label or 'False Negative Rate'
if self._far_at is not None:
self._trans_far_val = [ppndf(float(k)) for k in self._far_at]
#custom defaults here #custom defaults here
if self._x_rotation is None: if self._x_rotation is None:
self._x_rotation = 50 self._x_rotation = 50
...@@ -517,6 +513,14 @@ class Det(PlotBase): ...@@ -517,6 +513,14 @@ class Det(PlotBase):
linestyle=linestyle, linestyle=linestyle,
label=self._label('eval', eval_file, idx, **self._kwargs) label=self._label('eval', eval_file, idx, **self._kwargs)
) )
if self._far_at is not None:
from .. import farfrr
for line in self._far_at:
thres_line = far_threshold(dev_neg, dev_pos, line)
eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, thres_line)
eval_fmr, eval_fnmr = ppndf(eval_fmr), ppndf(eval_fnmr)
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
self._eval_points[line].append((eval_fmr, eval_fnmr))
else: else:
plot.det( plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx], dev_neg, dev_pos, self._points, color=self._colors[idx],
...@@ -542,6 +546,7 @@ class Epc(PlotBase): ...@@ -542,6 +546,7 @@ class Epc(PlotBase):
self._eval = True #always eval data with EPC self._eval = True #always eval data with EPC
self._split = False self._split = False
self._nb_figs = 1 self._nb_figs = 1
self._far_at = None
def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None): def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None):
''' Plot EPC using :py:func:`bob.measure.plot.epc` ''' ''' Plot EPC using :py:func:`bob.measure.plot.epc` '''
......
...@@ -125,7 +125,6 @@ CLI options ...@@ -125,7 +125,6 @@ CLI options
bob.measure.script.common_options.eval_option bob.measure.script.common_options.eval_option
bob.measure.script.common_options.sep_dev_eval_option bob.measure.script.common_options.sep_dev_eval_option
bob.measure.script.common_options.cmc_option bob.measure.script.common_options.cmc_option
bob.measure.script.common_options.semilogx_option
bob.measure.script.common_options.show_dev_option bob.measure.script.common_options.show_dev_option
bob.measure.script.common_options.print_filenames_option bob.measure.script.common_options.print_filenames_option
bob.measure.script.common_options.const_layout_option bob.measure.script.common_options.const_layout_option
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment