Commit 11a6e578 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Fix evaluate script (axes not set) and add defaults settings for the figure...

Fix evaluate script (axes not set) and add defaults settings for the figure classes. Condensate options: list of axis limits instead of individual min/max per axis.
parent 5540ada7
Pipeline #18416 failed with stage
in 26 minutes and 39 seconds
...@@ -7,17 +7,18 @@ from . import figure ...@@ -7,17 +7,18 @@ from . import figure
from . import common_options from . import common_options
from bob.extension.scripts.click_helper import verbosity_option from bob.extension.scripts.click_helper import verbosity_option
@click.command() @click.command()
@common_options.scores_argument(nargs=-1) @common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option() @common_options.test_option()
@common_options.table_option()
@common_options.open_file_mode_option() @common_options.open_file_mode_option()
@common_options.output_plot_metric_option() @common_options.output_plot_metric_option()
@common_options.criterion_option() @common_options.criterion_option()
@common_options.threshold_option() @common_options.threshold_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def metrics(ctx, scores, test, **kargs): def metrics(ctx, scores, test, **kwargs):
"""Prints a single output line that contains all info for a given """Prints a single output line that contains all info for a given
criterion (eer or hter). criterion (eer or hter).
...@@ -39,6 +40,7 @@ def metrics(ctx, scores, test, **kargs): ...@@ -39,6 +40,7 @@ def metrics(ctx, scores, test, **kargs):
process = figure.Metrics(ctx, scores, test, load.split_files) process = figure.Metrics(ctx, scores, test, load.split_files)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(nargs=-1) @common_options.scores_argument(nargs=-1)
@common_options.titles_option() @common_options.titles_option()
...@@ -47,16 +49,13 @@ def metrics(ctx, scores, test, **kargs): ...@@ -47,16 +49,13 @@ def metrics(ctx, scores, test, **kargs):
@common_options.test_option() @common_options.test_option()
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.semilogx_option(True) @common_options.semilogx_option(True)
@common_options.min_x_axis_val_option() @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.max_x_axis_val_option(dflt=1)
@common_options.min_y_axis_val_option(dflt=0)
@common_options.max_y_axis_val_option(dflt=1)
@common_options.axis_fontsize_option() @common_options.axis_fontsize_option()
@common_options.x_rotation_option() @common_options.x_rotation_option()
@common_options.fmr_line_at_option() @common_options.fmr_line_at_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def roc(ctx, scores, test, **kargs): def roc(ctx, scores, test, **kwargs):
"""Plot ROC (receiver operating characteristic) curve: """Plot ROC (receiver operating characteristic) curve:
The plot will represent the false match rate on the horizontal axis and the The plot will represent the false match rate on the horizontal axis and the
false non match rate on the vertical axis. The values for the axis will be false non match rate on the vertical axis. The values for the axis will be
...@@ -77,22 +76,20 @@ def roc(ctx, scores, test, **kargs): ...@@ -77,22 +76,20 @@ def roc(ctx, scores, test, **kargs):
process = figure.Roc(ctx, scores, test, load.split_files) process = figure.Roc(ctx, scores, test, load.split_files)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(nargs=-1) @common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf') @common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option() @common_options.titles_option()
@common_options.sep_dev_test_option() @common_options.sep_dev_test_option()
@common_options.test_option() @common_options.test_option()
@common_options.min_x_axis_val_option(dflt=0.01) @common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.max_x_axis_val_option(dflt=95)
@common_options.min_y_axis_val_option(dflt=0.01)
@common_options.max_y_axis_val_option(dflt=95)
@common_options.axis_fontsize_option(dflt=6) @common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45) @common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option() @common_options.points_curve_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def det(ctx, scores, test, **kargs): def det(ctx, scores, test, **kwargs):
"""Plot DET (detection error trade-off) curve: """Plot DET (detection error trade-off) curve:
modified ROC curve which plots error rates on both axes modified ROC curve which plots error rates on both axes
(false positives on the x-axis and false negatives on the y-axis) (false positives on the x-axis and false negatives on the y-axis)
...@@ -112,6 +109,7 @@ def det(ctx, scores, test, **kargs): ...@@ -112,6 +109,7 @@ def det(ctx, scores, test, **kargs):
process = figure.Det(ctx, scores, test, load.split_files) process = figure.Det(ctx, scores, test, load.split_files)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1) @common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf') @common_options.output_plot_file_option(default_out='epc.pdf')
...@@ -120,7 +118,7 @@ def det(ctx, scores, test, **kargs): ...@@ -120,7 +118,7 @@ def det(ctx, scores, test, **kargs):
@common_options.axis_fontsize_option() @common_options.axis_fontsize_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def epc(ctx, scores, **kargs): def epc(ctx, scores, **kwargs):
"""Plot EPC (expected performance curve): """Plot EPC (expected performance curve):
plots the error rate on the test set depending on a threshold selected plots the error rate on the test set depending on a threshold selected
a-priori on the development set and accounts for varying relative cost a-priori on the development set and accounts for varying relative cost
...@@ -137,6 +135,7 @@ def epc(ctx, scores, **kargs): ...@@ -137,6 +135,7 @@ def epc(ctx, scores, **kargs):
process = figure.Epc(ctx, scores, True, load.split_files) process = figure.Epc(ctx, scores, True, load.split_files)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(nargs=-1) @common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf') @common_options.output_plot_file_option(default_out='hist.pdf')
...@@ -147,7 +146,7 @@ def epc(ctx, scores, **kargs): ...@@ -147,7 +146,7 @@ def epc(ctx, scores, **kargs):
@common_options.threshold_option() @common_options.threshold_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def hist(ctx, scores, test, **kargs): def hist(ctx, scores, test, **kwargs):
""" Plots histograms of positive and negatives along with threshold """ Plots histograms of positive and negatives along with threshold
criterion. criterion.
...@@ -166,6 +165,7 @@ def hist(ctx, scores, test, **kargs): ...@@ -166,6 +165,7 @@ def hist(ctx, scores, test, **kargs):
process = figure.Hist(ctx, scores, test, load.split_files) process = figure.Hist(ctx, scores, test, load.split_files)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(nargs=-1) @common_options.scores_argument(nargs=-1)
@common_options.titles_option() @common_options.titles_option()
...@@ -177,10 +177,13 @@ def hist(ctx, scores, test, **kargs): ...@@ -177,10 +177,13 @@ def hist(ctx, scores, test, **kargs):
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.semilogx_option(dflt=True) @common_options.semilogx_option(dflt=True)
@common_options.n_bins_option() @common_options.n_bins_option()
@common_options.fmr_line_at_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def evaluate(ctx, scores, test, **kargs): def evaluate(ctx, scores, test, **kwargs):
'''Runs error analysis on score sets '''Runs error analysis on score sets
\b
1. Computes the threshold using either EER or min. HTER criteria on 1. Computes the threshold using either EER or min. HTER criteria on
development set scores development set scores
2. Applies the above threshold on test set scores to compute the HTER, if a 2. Applies the above threshold on test set scores to compute the HTER, if a
...@@ -191,6 +194,7 @@ def evaluate(ctx, scores, test, **kargs): ...@@ -191,6 +194,7 @@ def evaluate(ctx, scores, test, **kargs):
You need to provide 2 score files for each biometric system in this order: You need to provide 2 score files for each biometric system in this order:
\b \b
* development scores * development scores
* evaluation scores * evaluation scores
...@@ -200,31 +204,46 @@ def evaluate(ctx, scores, test, **kargs): ...@@ -200,31 +204,46 @@ def evaluate(ctx, scores, test, **kargs):
$ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores $ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores
''' '''
#first time erase if existing file # first time erase if existing file
click.echo("Computing metrics with EER...") click.echo("Computing metrics with EER...")
ctx.meta['criter'] = 'eer' #no criterion passed to evaluate ctx.meta['criter'] = 'eer' # no criterion passed to evaluate
ctx.invoke(metrics, scores=scores, test=test) ctx.invoke(metrics, scores=scores, test=test)
#second time, appends the content # second time, appends the content
click.echo("Computing metrics with HTER...") click.echo("Computing metrics with HTER...")
ctx.meta['criter'] = 'hter' #no criterion passed in evaluate ctx.meta['criter'] = 'hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, test=test) ctx.invoke(metrics, scores=scores, test=test)
if 'log' in ctx.meta: if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log']) click.echo("[metrics] => %s" % ctx.meta['log'])
#avoid closing pdf file before all figures are plotted # avoid closing pdf file before all figures are plotted
ctx.meta['closef'] = False ctx.meta['closef'] = False
if test: if test:
click.echo("Starting evaluate with dev and test scores...") click.echo("Starting evaluate with dev and test scores...")
else: else:
click.echo("Starting evaluate with dev scores only...") click.echo("Starting evaluate with dev scores only...")
click.echo("Computing ROC...") click.echo("Computing ROC...")
ctx.forward(roc) # set axes limits for ROC
ctx.forward(roc) # use class defaults plot settings
click.echo("Computing DET...") click.echo("Computing DET...")
ctx.forward(det) ctx.forward(det) # use class defaults plot settings
if test: if test:
click.echo("Computing EPC...") click.echo("Computing EPC...")
<<<<<<< HEAD
ctx.forward(epc) # use class defaults plot settings
# the last one closes the file
||||||| merged common ancestors
ctx.forward(epc)
<<<<<<< HEAD
#the last one closes the file
=======
ctx.forward(epc) ctx.forward(epc)
# the last one closes the file
>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
||||||| merged common ancestors
#the last one closes the file #the last one closes the file
=======
# the last one closes the file
>>>>>>> b98021d93c81eee46a730271fad67e001bf084ac
ctx.meta['closef'] = True ctx.meta['closef'] = True
click.echo("Computing score histograms...") click.echo("Computing score histograms...")
ctx.forward(hist) ctx.forward(hist)
......
...@@ -29,7 +29,8 @@ def scores_argument(test_mandatory=False, **kwargs): ...@@ -29,7 +29,8 @@ def scores_argument(test_mandatory=False, **kwargs):
' is on t') ' is on t')
raise click.BadParameter( raise click.BadParameter(
'%sest-score(s) must ' '%sest-score(s) must '
'be provided along with dev-score(s)' % pref, ctx=ctx) 'be provided along with dev-score(s)' % pref, ctx=ctx
)
else: else:
ctx.meta['dev-scores'] = [value[i] for i in ctx.meta['dev-scores'] = [value[i] for i in
range(length) if not i % 2] range(length) if not i % 2]
...@@ -86,57 +87,38 @@ def semilogx_option(dflt= False, **kwargs): ...@@ -86,57 +87,38 @@ def semilogx_option(dflt= False, **kwargs):
callback=callback, **kwargs)(func) callback=callback, **kwargs)(func)
return custom_semilogx_option return custom_semilogx_option
def min_x_axis_val_option(dflt=1e-4, **kwargs): def axes_val_option(dflt=None, **kwargs):
'''Get option for min value on x axis''' '''Get option for min/max values for axes. If one the default is None, no
def custom_min_x_axis_val_option(func): default is used
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_x'] = value
return value
return click.option(
'--min-x', type=float, default=dflt, show_default=True,
help='Min value display on X axis',
callback=callback, **kwargs)(func)
return custom_min_x_axis_val_option
def max_x_axis_val_option(dflt=1.0, **kwargs): Parameters
'''Get option for max value on x axis''' ----------
def custom_max_x_axis_val_option(func):
def callback(ctx, param, value):
ctx.meta['max_x'] = value
return value
return click.option(
'--max-x', type=float, default=dflt, show_default=True,
help='Max value display on X axis',
callback=callback, **kwargs)(func)
return custom_max_x_axis_val_option
def min_y_axis_val_option(dflt=1e-4, **kwargs):
'''Get option for min value on y axis'''
def custom_min_y_axis_val_option(func):
def callback(ctx, param, value):
if value > 0:
value = pow(10, int(math.floor(math.log(value, 10))))
ctx.meta['min_y'] = value
return value
return click.option(
'--min-y', type=float, default=dflt, show_default=True,
help='Min value display on Y axis',
callback=callback, **kwargs)(func)
return custom_min_y_axis_val_option
def max_y_axis_val_option(dflt=1.0, **kwargs): dflt: :any:`list`
'''Get option for max value on y axis''' List of default min/max values for axes. Must be of length 4
def custom_max_y_axis_val_option(func): '''
def custom_axes_val_option(func):
def callback(ctx, param, value): def callback(ctx, param, value):
ctx.meta['max_y'] = value if value is not None:
tmp = value.split(',')
if len(tmp) != 4:
raise click.BadParameter('Must provide 4 axis limits')
try:
value = [float(i) for i in tmp]
except:
raise click.BadParameter('Axis limits must be floats')
if None in value:
value = None
elif None not in dflt or len(dflt) == 4:
value = dflt if not all(isinstance(x, float) for x in dflt) else None
ctx.meta['axlim'] = value
return value return value
return click.option( return click.option(
'--max-y', type=float, default=dflt, show_default=True, '-L', '--axlim', default=None, show_default=True,
help='Max value display on Y axis', help='min/max axes values separated by commas (min_x, max_x, '
'min_y, max_y)',
callback=callback, **kwargs)(func) callback=callback, **kwargs)(func)
return custom_max_y_axis_val_option return custom_axes_val_option
def axis_fontsize_option(dflt=8, **kwargs): def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size''' '''Get option for axis font size'''
...@@ -328,7 +310,7 @@ def threshold_option(**kwargs): ...@@ -328,7 +310,7 @@ def threshold_option(**kwargs):
return custom_threshold_option return custom_threshold_option
def label_option(name_option='x-label', **kwargs): def label_option(name_option='x_label', **kwargs):
'''Get labels options based on the given name. '''Get labels options based on the given name.
Parameters: Parameters:
......
...@@ -319,17 +319,8 @@ class PlotBase(MeasureBase): ...@@ -319,17 +319,8 @@ class PlotBase(MeasureBase):
_split: :obj:`bool` _split: :obj:`bool`
If False, dev and test curves will be printed on the some figure If False, dev and test curves will be printed on the some figure
_min_x: :obj:`float` _axlim: :any:`list`
Minimum value for the X-axis Minimum/Maximum values for the X and Y axes
_max_x: :obj:`float`
Maximum value for the X-axis
_min_y: :obj:`float`
Minimum value for the Y-axis
_max_y: :obj:`float`
Maximum value for the Y-axis
_x_rotation: :obj:`int` _x_rotation: :obj:`int`
Rotation of the X axis labels Rotation of the X axis labels
...@@ -343,13 +334,10 @@ class PlotBase(MeasureBase): ...@@ -343,13 +334,10 @@ class PlotBase(MeasureBase):
self._points = None if 'points' not in ctx.meta else ctx.meta['points'] self._points = None if 'points' not in ctx.meta else ctx.meta['points']
self._titles = None if 'titles' not in ctx.meta else ctx.meta['titles'] self._titles = None if 'titles' not in ctx.meta else ctx.meta['titles']
self._split = None if 'split' not in ctx.meta else ctx.meta['split'] self._split = None if 'split' not in ctx.meta else ctx.meta['split']
self._min_x = None if 'min_x' not in ctx.meta else ctx.meta['min_x'] self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._min_y = None if 'min_y' not in ctx.meta else ctx.meta['min_y']
self._max_x = None if 'max_x' not in ctx.meta else ctx.meta['max_x']
self._max_y = None if 'max_y' not in ctx.meta else ctx.meta['max_y']
self._x_rotation = None if 'x_rotation' not in ctx.meta else \ self._x_rotation = None if 'x_rotation' not in ctx.meta else \
ctx.meta['x_rotation'] ctx.meta['x_rotation']
self._axisfontsize = None if 'fontsize' not in ctx.meta else \ self._axisfontsize = 6 if 'fontsize' not in ctx.meta else \
ctx.meta['fontsize'] ctx.meta['fontsize']
self._nb_figs = 2 if self._test and self._split else 1 self._nb_figs = 2 if self._test and self._split else 1
...@@ -371,11 +359,13 @@ class PlotBase(MeasureBase): ...@@ -371,11 +359,13 @@ class PlotBase(MeasureBase):
self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \ self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \
self._ctx.meta else PdfPages(self._output) self._ctx.meta else PdfPages(self._output)
mpl.figure(1) for i in range(self._nb_figs):
fig = mpl.figure(i + 1)
fig.clear()
if self._axisfontsize is not None: if self._axisfontsize is not None:
mpl.rc('xtick', labelsize=self._axisfontsize) mpl.rc('xtick', labelsize=self._axisfontsize)
mpl.rc('ytick', labelsize=self._axisfontsize) mpl.rc('ytick', labelsize=self._axisfontsize)
mpl.rc('legend', fontsize=7)
def end_process(self): def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and ''' Set title, legend, axis labels, grid colors, save figures and
...@@ -416,9 +406,10 @@ class PlotBase(MeasureBase): ...@@ -416,9 +406,10 @@ class PlotBase(MeasureBase):
return base + (" (%s)" % name) return base + (" (%s)" % name)
def _set_axis(self): def _set_axis(self):
axis = [self._min_x, self._max_x, self._min_y, self._max_y] if self._axlim is not None and None not in self._axlim:
if None not in axis: mpl.axis(self._axlim)
mpl.axis(axis) else:
mpl.axes().autoscale()
class Roc(PlotBase): class Roc(PlotBase):
''' Handles the plotting of ROC ''' Handles the plotting of ROC
...@@ -438,6 +429,9 @@ class Roc(PlotBase): ...@@ -438,6 +429,9 @@ class Roc(PlotBase):
self._title = 'ROC' self._title = 'ROC'
self._x_label = 'FMR' self._x_label = 'FMR'
self._y_label = ("1 - FNMR" if self._semilogx else "FNMR") self._y_label = ("1 - FNMR" if self._semilogx else "FNMR")
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None, def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None): test_neg=None, test_pos=None, test_fta=None, test_file=None):
...@@ -488,6 +482,9 @@ class Det(PlotBase): ...@@ -488,6 +482,9 @@ class Det(PlotBase):
def __init__(self, ctx, scores, test, func_load): def __init__(self, ctx, scores, test, func_load):
super(Det, self).__init__(ctx, scores, test, func_load) super(Det, self).__init__(ctx, scores, test, func_load)
self._title = 'DET' self._title = 'DET'
#custom defaults here
if self._x_rotation is None:
self._x_rotation = 50
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None, def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None): test_neg=None, test_pos=None, test_fta=None, test_file=None):
...@@ -517,9 +514,10 @@ class Det(PlotBase): ...@@ -517,9 +514,10 @@ class Det(PlotBase):
) )
def _set_axis(self): def _set_axis(self):
axis = [self._min_x, self._max_x, self._min_y, self._max_y] if self._axlim is not None and None not in self._axlim:
if None not in axis: plot.det_axis(self._axlim)
plot.det_axis(axis) else:
plot.det_axis([0.01, 99, 0.01, 99])
class Epc(PlotBase): class Epc(PlotBase):
''' Handles the plotting of EPC ''' ''' Handles the plotting of EPC '''
...@@ -531,6 +529,7 @@ class Epc(PlotBase): ...@@ -531,6 +529,7 @@ class Epc(PlotBase):
self._x_label = 'Cost' self._x_label = 'Cost'
self._y_label = 'Min. HTER (%)' self._y_label = 'Min. HTER (%)'
self._test = True #always test data with EPC self._test = True #always test data with EPC
self._nb_figs = 1
def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None, def compute(self, idx, dev_neg, dev_pos, dev_fta=None, dev_file=None,
test_neg=None, test_pos=None, test_fta=None, test_file=None): test_neg=None, test_pos=None, test_fta=None, test_file=None):
...@@ -605,7 +604,7 @@ class Hist(PlotBase): ...@@ -605,7 +604,7 @@ class Hist(PlotBase):
ax = mpl.gca() ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([]) ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01), mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
fontsize=10) fontsize=6)
else: else:
mpl.legend(loc='best', fancybox=True, framealpha=0.5) mpl.legend(loc='best', fancybox=True, framealpha=0.5)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment