From 11a6e57805bb421527a92f87a138e1107567da01 Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Thu, 5 Apr 2018 16:19:07 +0200
Subject: [PATCH] Fix evaluate script (axes not set) and add defaults settings
for the figure classes. Condensate options: list of axis limits instead of
individual min/max per axis.
---
bob/measure/script/commands.py | 69 ++++++++++++++++---------
bob/measure/script/common_options.py | 76 +++++++++++-----------------
bob/measure/script/figure.py | 55 ++++++++++----------
3 files changed, 100 insertions(+), 100 deletions(-)
diff --git a/bob/measure/script/commands.py b/bob/measure/script/commands.py
index 0d3fcaa..19de02b 100644
--- a/bob/measure/script/commands.py
+++ b/bob/measure/script/commands.py
@@ -7,17 +7,18 @@ from . import figure
from . import common_options
from bob.extension.scripts.click_helper import verbosity_option
+
@click.command()
@common_options.scores_argument(nargs=-1)
-@common_options.table_option()
@common_options.test_option()
+@common_options.table_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
-def metrics(ctx, scores, test, **kargs):
+def metrics(ctx, scores, test, **kwargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
@@ -39,6 +40,7 @@ def metrics(ctx, scores, test, **kargs):
process = figure.Metrics(ctx, scores, test, load.split_files)
process.run()
+
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@@ -47,16 +49,13 @@ def metrics(ctx, scores, test, **kargs):
@common_options.test_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
-@common_options.min_x_axis_val_option()
-@common_options.max_x_axis_val_option(dflt=1)
-@common_options.min_y_axis_val_option(dflt=0)
-@common_options.max_y_axis_val_option(dflt=1)
+@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
-def roc(ctx, scores, test, **kargs):
+def roc(ctx, scores, test, **kwargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
@@ -77,22 +76,20 @@ def roc(ctx, scores, test, **kargs):
process = figure.Roc(ctx, scores, test, load.split_files)
process.run()
+
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.test_option()
-@common_options.min_x_axis_val_option(dflt=0.01)
-@common_options.max_x_axis_val_option(dflt=95)
-@common_options.min_y_axis_val_option(dflt=0.01)
-@common_options.max_y_axis_val_option(dflt=95)
+@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@verbosity_option()
@click.pass_context
-def det(ctx, scores, test, **kargs):
+def det(ctx, scores, test, **kwargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
@@ -112,6 +109,7 @@ def det(ctx, scores, test, **kargs):
process = figure.Det(ctx, scores, test, load.split_files)
process.run()
+
@click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@@ -120,7 +118,7 @@ def det(ctx, scores, test, **kargs):
@common_options.axis_fontsize_option()
@verbosity_option()
@click.pass_context
-def epc(ctx, scores, **kargs):
+def epc(ctx, scores, **kwargs):
"""Plot EPC (expected performance curve):
plots the error rate on the test set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost
@@ -137,6 +135,7 @@ def epc(ctx, scores, **kargs):
process = figure.Epc(ctx, scores, True, load.split_files)
process.run()
+
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@@ -147,7 +146,7 @@ def epc(ctx, scores, **kargs):
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
-def hist(ctx, scores, test, **kargs):
+def hist(ctx, scores, test, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
@@ -166,6 +165,7 @@ def hist(ctx, scores, test, **kargs):
process = figure.Hist(ctx, scores, test, load.split_files)
process.run()
+
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@@ -177,20 +177,24 @@ def hist(ctx, scores, test, **kargs):
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
+@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
-def evaluate(ctx, scores, test, **kargs):
+def evaluate(ctx, scores, test, **kwargs):
'''Runs error analysis on score sets
+
+ \b
1. Computes the threshold using either EER or min. HTER criteria on
- development set scores
+ development set scores
2. Applies the above threshold on test set scores to compute the HTER, if a
- test-score set is provided
+ test-score set is provided
3. Reports error rates on the console
4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
- file (unless --no-plot is passed)
+ file (unless --no-plot is passed)
You need to provide 2 score files for each biometric system in this order:
+
\b
* development scores
* evaluation scores
@@ -200,31 +204,46 @@ def evaluate(ctx, scores, test, **kargs):
$ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores
'''
- #first time erase if existing file
+ # first time erase if existing file
click.echo("Computing metrics with EER...")
- ctx.meta['criter'] = 'eer' #no criterion passed to evaluate
+ ctx.meta['criter'] = 'eer' # no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, test=test)
- #second time, appends the content
+ # second time, appends the content
click.echo("Computing metrics with HTER...")
- ctx.meta['criter'] = 'hter' #no criterion passed in evaluate
+ ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, test=test)
if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log'])
- #avoid closing pdf file before all figures are plotted
+ # avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False
if test:
click.echo("Starting evaluate with dev and test scores...")
else:
click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...")
- ctx.forward(roc)
+ # set axes limits for ROC
+ ctx.forward(roc) # use class defaults plot settings
click.echo("Computing DET...")
- ctx.forward(det)
+ ctx.forward(det) # use class defaults plot settings
if test:
click.echo("Computing EPC...")
+<<<<<<< HEAD
+ ctx.forward(epc) # use class defaults plot settings
+ # the last one closes the file
+||||||| merged common ancestors
+ ctx.forward(epc)
+<<<<<<< HEAD
+ #the last one closes the file
+=======
ctx.forward(epc)
+ # the last one closes the file
+>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
+||||||| merged common ancestors
#the last one closes the file
+=======
+ # the last one closes the file
+>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
ctx.meta['closef'] = True
click.echo("Computing score histograms...")
ctx.forward(hist)
diff --git a/bob/measure/script/common_options.py b/bob/measure/script/common_options.py
index 83c47f6..d2d0a8c 100644
--- a/bob/measure/script/common_options.py
+++ b/bob/measure/script/common_options.py
@@ -29,7 +29,8 @@ def scores_argument(test_mandatory=False, **kwargs):
' is on t')
raise click.BadParameter(
'%sest-score(s) must '
- 'be provided along with dev-score(s)' % pref, ctx=ctx)
+ 'be provided along with dev-score(s)' % pref, ctx=ctx
+ )
else:
ctx.meta['dev-scores'] = [value[i] for i in
range(length) if not i % 2]
@@ -86,57 +87,38 @@ def semilogx_option(dflt= False, **kwargs):
callback=callback, **kwargs)(func)
return custom_semilogx_option
-def min_x_axis_val_option(dflt=1e-4, **kwargs):
- '''Get option for min value on x axis'''
- def custom_min_x_axis_val_option(func):
- def callback(ctx, param, value):
- if value > 0:
- value = pow(10, int(math.floor(math.log(value, 10))))
- ctx.meta['min_x'] = value
- return value
- return click.option(
- '--min-x', type=float, default=dflt, show_default=True,
- help='Min value display on X axis',
- callback=callback, **kwargs)(func)
- return custom_min_x_axis_val_option
-
-def max_x_axis_val_option(dflt=1.0, **kwargs):
- '''Get option for max value on x axis'''
- def custom_max_x_axis_val_option(func):
- def callback(ctx, param, value):
- ctx.meta['max_x'] = value
- return value
- return click.option(
- '--max-x', type=float, default=dflt, show_default=True,
- help='Max value display on X axis',
- callback=callback, **kwargs)(func)
- return custom_max_x_axis_val_option
+def axes_val_option(dflt=None, **kwargs):
+ '''Get option for min/max values for axes. If one the default is None, no
+ default is used
-def min_y_axis_val_option(dflt=1e-4, **kwargs):
- '''Get option for min value on y axis'''
- def custom_min_y_axis_val_option(func):
- def callback(ctx, param, value):
- if value > 0:
- value = pow(10, int(math.floor(math.log(value, 10))))
- ctx.meta['min_y'] = value
- return value
- return click.option(
- '--min-y', type=float, default=dflt, show_default=True,
- help='Min value display on Y axis',
- callback=callback, **kwargs)(func)
- return custom_min_y_axis_val_option
+ Parameters
+ ----------
-def max_y_axis_val_option(dflt=1.0, **kwargs):
- '''Get option for max value on y axis'''
- def custom_max_y_axis_val_option(func):
+ dflt: :any:`list`
+ List of default min/max values for axes. Must be of length 4
+ '''
+ def custom_axes_val_option(func):
def callback(ctx, param, value):
- ctx.meta['max_y'] = value
+ if value is not None:
+ tmp = value.split(',')
+ if len(tmp) != 4:
+ raise click.BadParameter('Must provide 4 axis limits')
+ try:
+ value = [float(i) for i in tmp]
+ except:
+ raise click.BadParameter('Axis limits must be floats')
+ if None in value:
+ value = None
+ elif None not in dflt or len(dflt) == 4:
+ value = dflt if not all(isinstance(x, float) for x in dflt) else None
+ ctx.meta['axlim'] = value
return value
return click.option(
- '--max-y', type=float, default=dflt, show_default=True,
- help='Max value display on Y axis',
+ '-L', '--axlim', default=None, show_default=True,
+ help='min/max axes values separated by commas (min_x, max_x, '
+ 'min_y, max_y)',
callback=callback, **kwargs)(func)
- return custom_max_y_axis_val_option
+ return custom_axes_val_option
def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size'''
@@ -328,7 +310,7 @@ def threshold_option(**kwargs):
return custom_threshold_option
-def label_option(name_option='x-label', **kwargs):
+def label_option(name_option='x_label', **kwargs):
'''Get labels options based on the given name.
Parameters:
diff --git a/bob/measure/script/figure.py b/bob/measure/script/figure.py
index 6ee3382..7c73b4d 100644
--- a/bob/measure/script/figure.py
+++ b/bob/measure/script/figure.py
@@ -319,17 +319,8 @@ class PlotBase(MeasureBase):
_split: :obj:`bool`
If False, dev and test curves will be printed on the some figure
- _min_x: :obj:`float`
- Minimum value for the X-axis
-
- _max_x: :obj:`float`
- Maximum value for the X-axis
-
- _min_y: :obj:`float`
- Minimum value for the Y-axis
-
- _max_y: :obj:`float`
- Maximum value for the Y-axis
+ _axlim: :any:`list`
+ Minimum/Maximum values for the X and Y axes
_x_rotation: :obj:`int`
Rotation of the X axis labels
@@ -343,13 +334,10 @@ class PlotBase(MeasureBase):
self._points = None if 'points' not in ctx.meta else ctx.meta['points']
self._titles = None if 'titles' not in ctx.meta else ctx.meta['titles']
self._split = None if 'split' not in ctx.meta else ctx.meta['split']
- self._min_x = None if 'min_x' not in ctx.meta else ctx.meta['min_x']
- self._min_y = None if 'min_y' not in ctx.meta else ctx.meta['min_y']
- self._max_x = None if 'max_x' not in ctx.meta else ctx.meta['max_x']
- self._max_y = None if 'max_y' not in ctx.meta else ctx.meta['max_y']
+ self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._x_rotation = None if 'x_rotation' not in ctx.meta else \
ctx.meta['x_rotation']
- self._axisfontsize = None if 'fontsize' not in ctx.meta else \
+ self._axisfontsize = 6 if 'fontsize' not in ctx.meta else \
ctx.meta['fontsize']
self._nb_figs = 2 if self._test and self._split else 1
@@ -371,11 +359,13 @@ class PlotBase(MeasureBase):
self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \
self._ctx.meta else PdfPages(self._output)
- mpl.figure(1)
-
- if self._axisfontsize is not None:
- mpl.rc('xtick', labelsize=self._axisfontsize)
- mpl.rc('ytick', labelsize=self._axisfontsize)
+ for i in range(self._nb_figs):
+ fig = mpl.figure(i + 1)
+ fig.clear()
+ if self._axisfontsize is not None:
+ mpl.rc('xtick', labelsize=self._axisfontsize)
+ mpl.rc('ytick', labelsize=self._axisfontsize)
+ mpl.rc('legend', fontsize=7)
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
@@ -416,9 +406,10 @@ class PlotBase(MeasureBase):
return base + (" (%s)" % name)
def _set_axis(self):
- axis = [self._min_x, self._max_x, self._min_y, self._max_y]
- if None not in axis:
- mpl.axis(axis)
+ if self._axlim is not None and None not in self._axlim:
+ mpl.axis(self._axlim)
+ else:
+ mpl.axes().autoscale()
class Roc(PlotBase):
''' Handles the plotting of ROC
@@ -438,6 +429,9 @@ class Roc(PlotBase):
self._title = 'ROC'
self._x_label = 'FMR'
self._y_label = ("1 - FNMR" if self._semilogx else "FNMR")
+ #custom defaults
+ if self._axlim is None:
+ self._axlim = [1e-4, 1.0, 1e-4, 1.0]
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
@@ -488,6 +482,9 @@ class Det(PlotBase):
def __init__(self, ctx, scores, test, func_load):
super(Det, self).__init__(ctx, scores, test, func_load)
self._title = 'DET'
+ #custom defaults here
+ if self._x_rotation is None:
+ self._x_rotation = 50
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
@@ -517,9 +514,10 @@ class Det(PlotBase):
)
def _set_axis(self):
- axis = [self._min_x, self._max_x, self._min_y, self._max_y]
- if None not in axis:
- plot.det_axis(axis)
+ if self._axlim is not None and None not in self._axlim:
+ plot.det_axis(self._axlim)
+ else:
+ plot.det_axis([0.01, 99, 0.01, 99])
class Epc(PlotBase):
''' Handles the plotting of EPC '''
@@ -531,6 +529,7 @@ class Epc(PlotBase):
self._x_label = 'Cost'
self._y_label = 'Min. HTER (%)'
self._test = True #always test data with EPC
+ self._nb_figs = 1
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
@@ -605,7 +604,7 @@ class Hist(PlotBase):
ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
- fontsize=10)
+ fontsize=6)
else:
mpl.legend(loc='best', fancybox=True, framealpha=0.5)
--
GitLab