Commit 11a6e578 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Fix evaluate script (axes not set) and add defaults settings for the figure...

Fix evaluate script (axes not set) and add defaults settings for the figure classes. Condensate options: list of axis limits instead of individual min/max per axis.
parent 5540ada7
Pipeline #18416 failed with stage
in 26 minutes and 39 seconds
......@@ -7,17 +7,18 @@ from . import figure
from . import common_options
from bob.extension.scripts.click_helper import verbosity_option
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.table_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, test, **kargs):
def metrics(ctx, scores, test, **kwargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
......@@ -39,6 +40,7 @@ def metrics(ctx, scores, test, **kargs):
process = figure.Metrics(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
......@@ -47,16 +49,13 @@ def metrics(ctx, scores, test, **kargs):
@common_options.test_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.min_x_axis_val_option()
@common_options.max_x_axis_val_option(dflt=1)
@common_options.min_y_axis_val_option(dflt=0)
@common_options.max_y_axis_val_option(dflt=1)
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, test, **kargs):
def roc(ctx, scores, test, **kwargs):
"""Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be
......@@ -77,22 +76,20 @@ def roc(ctx, scores, test, **kargs):
process = figure.Roc(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.test_option()
@common_options.min_x_axis_val_option(dflt=0.01)
@common_options.max_x_axis_val_option(dflt=95)
@common_options.min_y_axis_val_option(dflt=0.01)
@common_options.max_y_axis_val_option(dflt=95)
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, test, **kargs):
def det(ctx, scores, test, **kwargs):
"""Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis)
......@@ -112,6 +109,7 @@ def det(ctx, scores, test, **kargs):
process = figure.Det(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
......@@ -120,7 +118,7 @@ def det(ctx, scores, test, **kargs):
@common_options.axis_fontsize_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kargs):
def epc(ctx, scores, **kwargs):
"""Plot EPC (expected performance curve):
plots the error rate on the test set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost
......@@ -137,6 +135,7 @@ def epc(ctx, scores, **kargs):
process = figure.Epc(ctx, scores, True, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
......@@ -147,7 +146,7 @@ def epc(ctx, scores, **kargs):
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kargs):
def hist(ctx, scores, test, **kwargs):
""" Plots histograms of positive and negatives along with threshold
criterion.
......@@ -166,6 +165,7 @@ def hist(ctx, scores, test, **kargs):
process = figure.Hist(ctx, scores, test, load.split_files)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
......@@ -177,20 +177,24 @@ def hist(ctx, scores, test, **kargs):
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, test, **kargs):
def evaluate(ctx, scores, test, **kwargs):
'''Runs error analysis on score sets
\b
1. Computes the threshold using either EER or min. HTER criteria on
development set scores
development set scores
2. Applies the above threshold on test set scores to compute the HTER, if a
test-score set is provided
test-score set is provided
3. Reports error rates on the console
4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
file (unless --no-plot is passed)
file (unless --no-plot is passed)
You need to provide 2 score files for each biometric system in this order:
\b
* development scores
* evaluation scores
......@@ -200,31 +204,46 @@ def evaluate(ctx, scores, test, **kargs):
$ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores
'''
#first time erase if existing file
# first time erase if existing file
click.echo("Computing metrics with EER...")
ctx.meta['criter'] = 'eer' #no criterion passed to evaluate
ctx.meta['criter'] = 'eer' # no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, test=test)
#second time, appends the content
# second time, appends the content
click.echo("Computing metrics with HTER...")
ctx.meta['criter'] = 'hter' #no criterion passed in evaluate
ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, test=test)
if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log'])
#avoid closing pdf file before all figures are plotted
# avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False
if test:
click.echo("Starting evaluate with dev and test scores...")
else:
click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...")
ctx.forward(roc)
# set axes limits for ROC
ctx.forward(roc) # use class defaults plot settings
click.echo("Computing DET...")
ctx.forward(det)
ctx.forward(det) # use class defaults plot settings
if test:
click.echo("Computing EPC...")
<<<<<<< HEAD
ctx.forward(epc) # use class defaults plot settings
# the last one closes the file
||||||| merged common ancestors
ctx.forward(epc)
<<<<<<< HEAD
#the last one closes the file
=======
ctx.forward(epc)
# the last one closes the file
>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
||||||| merged common ancestors
#the last one closes the file
=======
# the last one closes the file
>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
ctx.meta['closef'] = True
click.echo("Computing score histograms...")
ctx.forward(hist)
......
......@@ -29,7 +29,8 @@ def scores_argument(test_mandatory=False, **kwargs):
' is on t')
raise click.BadParameter(
'%sest-score(s) must '
'be provided along with dev-score(s)' % pref, ctx=ctx)
'be provided along with dev-score(s)' % pref, ctx=ctx
)
else:
ctx.meta['dev-scores'] = [value[i] for i in
range(length) if not i % 2]
......@@ -86,57 +87,38 @@ def semilogx_option(dflt= False, **kwargs):
callback=callback, **kwargs)(func)
return custom_semilogx_option
def min_x_axis_val_option(dflt=1e-4, **kwargs):
'''Get option for min value on x axis'''
def custom_min_x_axis_val_option(func):
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_x'] = value
return value
return click.option(
'--min-x', type=float, default=dflt, show_default=True,
help='Min value display on X axis',
callback=callback, **kwargs)(func)
return custom_min_x_axis_val_option
def max_x_axis_val_option(dflt=1.0, **kwargs):
'''Get option for max value on x axis'''
def custom_max_x_axis_val_option(func):
def callback(ctx, param, value):
ctx.meta['max_x'] = value
return value
return click.option(
'--max-x', type=float, default=dflt, show_default=True,
help='Max value display on X axis',
callback=callback, **kwargs)(func)
return custom_max_x_axis_val_option
def axes_val_option(dflt=None, **kwargs):
'''Get option for min/max values for axes. If one the default is None, no
default is used
def min_y_axis_val_option(dflt=1e-4, **kwargs):
'''Get option for min value on y axis'''
def custom_min_y_axis_val_option(func):
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_y'] = value
return value
return click.option(
'--min-y', type=float, default=dflt, show_default=True,
help='Min value display on Y axis',
callback=callback, **kwargs)(func)
return custom_min_y_axis_val_option
Parameters
----------
def max_y_axis_val_option(dflt=1.0, **kwargs):
'''Get option for max value on y axis'''
def custom_max_y_axis_val_option(func):
dflt: :any:`list`
List of default min/max values for axes. Must be of length 4
'''
def custom_axes_val_option(func):
def callback(ctx, param, value):
ctx.meta['max_y'] = value
if value is not None:
tmp = value.split(',')
if len(tmp) != 4:
raise click.BadParameter('Must provide 4 axis limits')
try:
value = [float(i) for i in tmp]
except:
raise click.BadParameter('Axis limits must be floats')
if None in value:
value = None
elif None not in dflt or len(dflt) == 4:
value = dflt if not all(isinstance(x, float) for x in dflt) else None
ctx.meta['axlim'] = value
return value
return click.option(
'--max-y', type=float, default=dflt, show_default=True,
help='Max value display on Y axis',
'-L', '--axlim', default=None, show_default=True,
help='min/max axes values separated by commas (min_x, max_x, '
'min_y, max_y)',
callback=callback, **kwargs)(func)
return custom_max_y_axis_val_option
return custom_axes_val_option
def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size'''
......@@ -328,7 +310,7 @@ def threshold_option(**kwargs):
return custom_threshold_option
def label_option(name_option='x-label', **kwargs):
def label_option(name_option='x_label', **kwargs):
'''Get labels options based on the given name.
Parameters:
......
......@@ -319,17 +319,8 @@ class PlotBase(MeasureBase):
_split: :obj:`bool`
If False, dev and test curves will be printed on the some figure
_min_x: :obj:`float`
Minimum value for the X-axis
_max_x: :obj:`float`
Maximum value for the X-axis
_min_y: :obj:`float`
Minimum value for the Y-axis
_max_y: :obj:`float`
Maximum value for the Y-axis
_axlim: :any:`list`
Minimum/Maximum values for the X and Y axes
_x_rotation: :obj:`int`
Rotation of the X axis labels
......@@ -343,13 +334,10 @@ class PlotBase(MeasureBase):
self._points = None if 'points' not in ctx.meta else ctx.meta['points']
self._titles = None if 'titles' not in ctx.meta else ctx.meta['titles']
self._split = None if 'split' not in ctx.meta else ctx.meta['split']
self._min_x = None if 'min_x' not in ctx.meta else ctx.meta['min_x']
self._min_y = None if 'min_y' not in ctx.meta else ctx.meta['min_y']
self._max_x = None if 'max_x' not in ctx.meta else ctx.meta['max_x']
self._max_y = None if 'max_y' not in ctx.meta else ctx.meta['max_y']
self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._x_rotation = None if 'x_rotation' not in ctx.meta else \
ctx.meta['x_rotation']
self._axisfontsize = None if 'fontsize' not in ctx.meta else \
self._axisfontsize = 6 if 'fontsize' not in ctx.meta else \
ctx.meta['fontsize']
self._nb_figs = 2 if self._test and self._split else 1
......@@ -371,11 +359,13 @@ class PlotBase(MeasureBase):
self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \
self._ctx.meta else PdfPages(self._output)
mpl.figure(1)
if self._axisfontsize is not None:
mpl.rc('xtick', labelsize=self._axisfontsize)
mpl.rc('ytick', labelsize=self._axisfontsize)
for i in range(self._nb_figs):
fig = mpl.figure(i + 1)
fig.clear()
if self._axisfontsize is not None:
mpl.rc('xtick', labelsize=self._axisfontsize)
mpl.rc('ytick', labelsize=self._axisfontsize)
mpl.rc('legend', fontsize=7)
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
......@@ -416,9 +406,10 @@ class PlotBase(MeasureBase):
return base + (" (%s)" % name)
def _set_axis(self):
axis = [self._min_x, self._max_x, self._min_y, self._max_y]
if None not in axis:
mpl.axis(axis)
if self._axlim is not None and None not in self._axlim:
mpl.axis(self._axlim)
else:
mpl.axes().autoscale()
class Roc(PlotBase):
''' Handles the plotting of ROC
......@@ -438,6 +429,9 @@ class Roc(PlotBase):
self._title = 'ROC'
self._x_label = 'FMR'
self._y_label = ("1 - FNMR" if self._semilogx else "FNMR")
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
......@@ -488,6 +482,9 @@ class Det(PlotBase):
def __init__(self, ctx, scores, test, func_load):
super(Det, self).__init__(ctx, scores, test, func_load)
self._title = 'DET'
#custom defaults here
if self._x_rotation is None:
self._x_rotation = 50
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
......@@ -517,9 +514,10 @@ class Det(PlotBase):
)
def _set_axis(self):
axis = [self._min_x, self._max_x, self._min_y, self._max_y]
if None not in axis:
plot.det_axis(axis)
if self._axlim is not None and None not in self._axlim:
plot.det_axis(self._axlim)
else:
plot.det_axis([0.01, 99, 0.01, 99])
class Epc(PlotBase):
''' Handles the plotting of EPC '''
......@@ -531,6 +529,7 @@ class Epc(PlotBase):
self._x_label = 'Cost'
self._y_label = 'Min. HTER (%)'
self._test = True #always test data with EPC
self._nb_figs = 1
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None):
......@@ -605,7 +604,7 @@ class Hist(PlotBase):
ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
fontsize=10)
fontsize=6)
else:
mpl.legend(loc='best', fancybox=True, framealpha=0.5)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment