From 00a68cf546d7cbbd0d58a1f7229858238ea021b8 Mon Sep 17 00:00:00 2001 From: Theophile GENTILHOMME Date: Wed, 18 Apr 2018 15:42:56 +0200 Subject: [PATCH] Detailed option description, matplolib setting moved to callback functions, add vertical line option to DET --- bob/measure/plot.py | 2 +- bob/measure/script/commands.py | 3 +- bob/measure/script/common_options.py | 24 +++++-- bob/measure/script/figure.py | 95 +++++++++++++++------------- doc/py_api.rst | 1 - 5 files changed, 70 insertions(+), 55 deletions(-) diff --git a/bob/measure/plot.py b/bob/measure/plot.py index 1512448..536032b 100644 --- a/bob/measure/plot.py +++ b/bob/measure/plot.py @@ -142,7 +142,7 @@ def roc_for_far(negatives, positives, far_values=log_values(), **kwargs): from matplotlib import pyplot from . import roc_for_far as calc out = calc(negatives, positives, far_values) - return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs) + return pyplot.semilogx(out[0, :], (1 - out[1, :]), **kwargs) def precision_recall_curve(negatives, positives, npoints=100, **kwargs): diff --git a/bob/measure/script/commands.py b/bob/measure/script/commands.py index 2eddbed..427976f 100644 --- a/bob/measure/script/commands.py +++ b/bob/measure/script/commands.py @@ -48,7 +48,6 @@ def metrics(ctx, scores, evaluation, **kwargs): @common_options.output_plot_file_option(default_out='roc.pdf') @common_options.eval_option() @common_options.points_curve_option() -@common_options.semilogx_option(True) @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1]) @common_options.x_rotation_option() @common_options.x_label_option() @@ -92,6 +91,7 @@ def roc(ctx, scores, evaluation, **kwargs): @common_options.x_label_option() @common_options.y_label_option() @common_options.points_curve_option() +@common_options.lines_at_option() @common_options.const_layout_option() @common_options.figsize_option() @common_options.style_option() @@ -196,7 +196,6 @@ def hist(ctx, scores, evaluation, **kwargs): @common_options.output_plot_metric_option() @common_options.output_plot_file_option(default_out='eval_plots.pdf') @common_options.points_curve_option() -@common_options.semilogx_option(dflt=True) @common_options.n_bins_option() @common_options.lines_at_option() @common_options.const_layout_option() diff --git a/bob/measure/script/common_options.py b/bob/measure/script/common_options.py index 4d50835..8f941d3 100644 --- a/bob/measure/script/common_options.py +++ b/bob/measure/script/common_options.py @@ -106,13 +106,23 @@ def print_filenames_option(dflt=True, **kwargs): def const_layout_option(dflt=True, **kwargs): '''Option to set matplotlib constrained_layout''' - return bool_option('clayout', 'Y', '(De)Activate constrained layout', dflt) + def custom_layout_option(func): + def callback(ctx, param, value): + ctx.meta['clayout'] = value + plt.rcParams['figure.constrained_layout.use'] = value + return value + return click.option( + '-Y', '--clayout/--no-clayout', default=dflt, show_default=True, + help='(De)Activate constrained layout', + callback=callback, **kwargs)(func) + return custom_layout_option def axes_val_option(dflt=None, **kwargs): ''' Option for setting min/max values on axes ''' return list_float_option( name='axlim', short_name='L', - desc='min/max axes values separated by commas (min_x, max_x, min_y, max_y)', + desc='min/max axes values separated by commas (e.g. ``--axlim ' + ' 0.1,100,0.1,100``)', nitems=4, dflt=dflt, **kwargs ) @@ -120,7 +130,8 @@ def thresholds_option(**kwargs): ''' Option to give a list of thresholds ''' return list_float_option( name='thres', short_name='T', - desc='Given threshold for metrics computations', + desc='Given threshold for metrics computations, e.g. ' + '0.005,0.001,0.056', nitems=None, dflt=None, **kwargs ) @@ -128,7 +139,7 @@ def lines_at_option(**kwargs): '''Get option to draw const far line''' return list_float_option( name='lines-at', short_name='la', - desc='If given, draw veritcal lines on ROC plots', + desc='If given, draw veritcal lines at the given axis positions', nitems=None, dflt=None, **kwargs ) @@ -291,11 +302,12 @@ def figsize_option(**kwargs): def callback(ctx, param, value): ctx.meta['figsize'] = value if value is None else \ [float(x) for x in value.split(',')] - plt.figure(figsize=ctx.meta['figsize']) + if value is not None: + plt.rcParams['figure.figsize'] = ctx.meta['figsize'] return value return click.option( '--figsize', help='If given, will run ' - '``plt.figure(figsize=figsize)``. Example: --fig-size 4,6', + '``plt.rcParams[\'figure.figsize\']=figsize)``. Example: --fig-size 4,6', callback=callback, **kwargs)(func) return custom_figsize_option diff --git a/bob/measure/script/figure.py b/bob/measure/script/figure.py index af34015..3533917 100644 --- a/bob/measure/script/figure.py +++ b/bob/measure/script/figure.py @@ -9,8 +9,7 @@ import matplotlib import matplotlib.pyplot as mpl from matplotlib.backends.backend_pdf import PdfPages from tabulate import tabulate -from .. import plot -from .. import utils +from .. import (far_threshold, plot, utils, ppndf) LINESTYLES = [ (0, ()), #solid @@ -329,6 +328,12 @@ class PlotBase(MeasureBase): self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim'] self._clayout = None if 'clayout' not in ctx.meta else\ ctx.meta['clayout'] + self._far_at = None if 'lines_at' not in ctx.meta else\ + ctx.meta['lines_at'] + self._trans_far_val = self._far_at + if self._far_at is not None: + self._eval_points = {line: [] for line in self._far_at} + self._lines_val = [] self._print_fn = True if 'show_fn' not in ctx.meta else\ ctx.meta['show_fn'] self._x_rotation = None if 'x_rotation' not in ctx.meta else \ @@ -365,8 +370,28 @@ class PlotBase(MeasureBase): fig.clear() def end_process(self): - ''' Set title, legend, axis labels, grid colors, save figures and - close pdf is needed ''' + ''' Set title, legend, axis labels, grid colors, save figures, drow + lines and close pdf if needed ''' + #draw vertical lines + if self._far_at is not None: + for (line, line_trans) in zip(self._far_at, self._trans_far_val): + mpl.figure(1) + mpl.plot( + [line_trans, line_trans], [-100.0, 100.], "--", + color='black' + ) + if self._eval and self._split: + mpl.figure(2) + x_values = [i for i, _ in self._eval_points[line]] + y_values = [j for _, j in self._eval_points[line]] + sort_indice = sorted( + range(len(x_values)), key=x_values.__getitem__ + ) + x_values = [x_values[i] for i in sort_indice] + y_values = [y_values[i] for i in sort_indice] + mpl.plot(x_values, + y_values, '--', + color='black') #only for plots if self._end_setup_plot: for i in range(self._nb_figs): @@ -390,7 +415,6 @@ class PlotBase(MeasureBase): ('closef' not in self._ctx.meta or self._ctx.meta['closef']): self._pdf_page.close() - #common protected functions def _label(self, base, name, idx): @@ -408,20 +432,12 @@ class Roc(PlotBase): ''' Handles the plotting of ROC''' def __init__(self, ctx, scores, evaluation, func_load): super(Roc, self).__init__(ctx, scores, evaluation, func_load) - self._semilogx = True if 'semilogx' not in ctx.meta else\ - ctx.meta['semilogx'] - self._far_at = None if 'lines_at' not in ctx.meta else\ - ctx.meta['lines_at'] self._title = self._title or 'ROC' self._x_label = self._x_label or 'False Positive Rate' - self._y_label = self._y_label or ( - "1 - False Negative Rate" if self._semilogx else "False Negative Rate" - ) + self._y_label = self._y_label or "1 - False Negative Rate" #custom defaults if self._axlim is None: self._axlim = [1e-4, 1.0, 1e-4, 1.0] - if self._far_at is not None: - self._eval_points = {line: [] for line in self._far_at} def compute(self, idx, dev_score, dev_file=None, eval_score=None, eval_file=None): @@ -432,8 +448,8 @@ class Roc(PlotBase): mpl.figure(1) if self._eval: linestyle = '-' if not self._split else LINESTYLES[idx % 14] - plot.roc( - dev_neg, dev_pos, self._points, self._semilogx, + plot.roc_for_far( + dev_neg, dev_pos, color=self._colors[idx], linestyle=linestyle, label=self._label('development', dev_file, idx, **self._kwargs) ) @@ -442,48 +458,26 @@ class Roc(PlotBase): mpl.figure(2) linestyle = LINESTYLES[idx % 14] - plot.roc( - eval_neg, eval_pos, self._points, self._semilogx, + plot.roc_for_far( + eval_neg, eval_pos, color=self._colors[idx], linestyle=linestyle, label=self._label('eval', eval_file, idx, **self._kwargs) ) if self._far_at is not None: from .. import farfrr for line in self._far_at: - eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, line) - if self._semilogx: - eval_fnmr = 1 - eval_fnmr + thres_line = far_threshold(dev_neg, dev_pos, line) + eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, thres_line) + eval_fnmr = 1 - eval_fnmr mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30) self._eval_points[line].append((eval_fmr, eval_fnmr)) else: - plot.roc( - dev_neg, dev_pos, self._points, self._semilogx, + plot.roc_for_far( + dev_neg, dev_pos, color=self._colors[idx], linestyle=LINESTYLES[idx % 14], label=self._label('development', dev_file, idx, **self._kwargs) ) - def end_process(self): - ''' Draw vertical line on the dev plot at the given fmr and print the - corresponding points on the eval plot for all the systems ''' - #draw vertical lines - if self._far_at is not None: - for line in self._far_at: - mpl.figure(1) - mpl.plot([line, line], [0., 1.], "--", color='black') - if self._eval and self._split: - mpl.figure(2) - x_values = [i for i, _ in self._eval_points[line]] - y_values = [j for _, j in self._eval_points[line]] - sort_indice = sorted( - range(len(x_values)), key=x_values.__getitem__ - ) - x_values = [x_values[i] for i in sort_indice] - y_values = [y_values[i] for i in sort_indice] - mpl.plot(x_values, - y_values, '--', - color='black') - super(Roc, self).end_process() - class Det(PlotBase): ''' Handles the plotting of DET ''' def __init__(self, ctx, scores, evaluation, func_load): @@ -491,6 +485,8 @@ class Det(PlotBase): self._title = self._title or 'DET' self._x_label = self._x_label or 'False Positive Rate' self._y_label = self._y_label or 'False Negative Rate' + if self._far_at is not None: + self._trans_far_val = [ppndf(float(k)) for k in self._far_at] #custom defaults here if self._x_rotation is None: self._x_rotation = 50 @@ -517,6 +513,14 @@ class Det(PlotBase): linestyle=linestyle, label=self._label('eval', eval_file, idx, **self._kwargs) ) + if self._far_at is not None: + from .. import farfrr + for line in self._far_at: + thres_line = far_threshold(dev_neg, dev_pos, line) + eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, thres_line) + eval_fmr, eval_fnmr = ppndf(eval_fmr), ppndf(eval_fnmr) + mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30) + self._eval_points[line].append((eval_fmr, eval_fnmr)) else: plot.det( dev_neg, dev_pos, self._points, color=self._colors[idx], @@ -542,6 +546,7 @@ class Epc(PlotBase): self._eval = True #always eval data with EPC self._split = False self._nb_figs = 1 + self._far_at = None def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None): ''' Plot EPC using :py:func:`bob.measure.plot.epc` ''' diff --git a/doc/py_api.rst b/doc/py_api.rst index 7bb8546..393d088 100644 --- a/doc/py_api.rst +++ b/doc/py_api.rst @@ -125,7 +125,6 @@ CLI options bob.measure.script.common_options.eval_option bob.measure.script.common_options.sep_dev_eval_option bob.measure.script.common_options.cmc_option - bob.measure.script.common_options.semilogx_option bob.measure.script.common_options.show_dev_option bob.measure.script.common_options.print_filenames_option bob.measure.script.common_options.const_layout_option -- 2.21.0