Commit 1c6e5bca authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Add min-far-value option, rename -l (already used) to -Z, change the way...

Add min-far-value option, rename -l (already used) to -Z, change the way linestyles are handled and make use of them optional
parent 77f11de1
Pipeline #19647 passed with stage
in 40 minutes and 8 seconds
...@@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs): ...@@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs):
# these are some constants required in this method # these are some constants required in this method
desiredTicks = [ desiredTicks = [
"0.000001", "0.000002", "0.000005",
"0.00001", "0.00002", "0.00005", "0.00001", "0.00002", "0.00005",
"0.0001", "0.0002", "0.0005", "0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005", "0.001", "0.002", "0.005",
...@@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs): ...@@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs):
] ]
desiredLabels = [ desiredLabels = [
"0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005", "0.001", "0.002", "0.005",
"0.01", "0.02", "0.05", "0.01", "0.02", "0.05",
"0.1", "0.2", "0.5", "0.1", "0.2", "0.5",
......
...@@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs): ...@@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.eval_option() @common_options.eval_option()
@common_options.points_curve_option() @common_options.points_curve_option()
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1]) @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.min_far_option()
@common_options.x_rotation_option() @common_options.x_rotation_option()
@common_options.x_label_option() @common_options.x_label_option()
@common_options.y_label_option() @common_options.y_label_option()
...@@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs): ...@@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
@common_options.linestyles_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def roc(ctx, scores, evaluation, **kwargs): def roc(ctx, scores, evaluation, **kwargs):
...@@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs): ...@@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.sep_dev_eval_option() @common_options.sep_dev_eval_option()
@common_options.eval_option() @common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95]) @common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.min_far_option()
@common_options.x_rotation_option(dflt=45) @common_options.x_rotation_option(dflt=45)
@common_options.x_label_option() @common_options.x_label_option()
@common_options.y_label_option() @common_options.y_label_option()
...@@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs): ...@@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
@common_options.linestyles_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def det(ctx, scores, evaluation, **kwargs): def det(ctx, scores, evaluation, **kwargs):
...@@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs): ...@@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs):
@common_options.y_label_option() @common_options.y_label_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
@common_options.linestyles_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def epc(ctx, scores, **kwargs): def epc(ctx, scores, **kwargs):
...@@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs): ...@@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs):
@common_options.legends_option() @common_options.legends_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
@common_options.linestyles_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def hist(ctx, scores, evaluation, **kwargs): def hist(ctx, scores, evaluation, **kwargs):
...@@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.figsize_option() @common_options.figsize_option()
@common_options.style_option() @common_options.style_option()
@common_options.linestyles_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def evaluate(ctx, scores, evaluation, **kwargs): def evaluate(ctx, scores, evaluation, **kwargs):
......
...@@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs): ...@@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs):
dflt dflt
) )
def linestyles_option(dflt=False, **kwargs):
''' Get option flag to turn on/off linestyles'''
return bool_option('line-linestyles', 'S', 'If given, applies a different '
'linestyles to each line.', dflt, **kwargs)
def cmc_option(**kwargs): def cmc_option(**kwargs):
'''Get option flag to say if cmc scores''' '''Get option flag to say if cmc scores'''
return bool_option('cmc', 'C', 'If set, CMC score files are provided') return bool_option('cmc', 'C', 'If set, CMC score files are provided',
**kwargs)
def semilogx_option(dflt=False, **kwargs): def semilogx_option(dflt=False, **kwargs):
'''Option to use semilog X-axis''' '''Option to use semilog X-axis'''
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt) return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt,
**kwargs)
def show_dev_option(dflt=False, **kwargs): def show_dev_option(dflt=False, **kwargs):
'''Option to tell if should show dev histo''' '''Option to tell if should show dev histo'''
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt) return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt,
**kwargs)
def print_filenames_option(dflt=True, **kwargs): def print_filenames_option(dflt=True, **kwargs):
'''Option to tell if filenames should be in the title''' '''Option to tell if filenames should be in the title'''
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt) return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt,
**kwargs)
def const_layout_option(dflt=True, **kwargs): def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout''' '''Option to set matplotlib constrained_layout'''
...@@ -278,6 +287,21 @@ def far_option(**kwargs): ...@@ -278,6 +287,21 @@ def far_option(**kwargs):
callback=callback, show_default=True,**kwargs)(func) callback=callback, show_default=True,**kwargs)(func)
return custom_far_option return custom_far_option
def min_far_option(dflt=1e-4, **kwargs):
'''Get option to get min far value'''
def custom_min_far_option(func):
def callback(ctx, param, value):
if value is not None and (value > 1 or value < 0):
raise click.BadParameter("FAR value should be between 0 and 1")
ctx.meta['min_far_value'] = value
return value
return click.option(
'-M', '--min-far-value', type=click.FLOAT, default=dflt,
help='Select the minimum FAR value used in ROC and DET plots; '
'should be a power of 10.',
callback=callback, show_default=True,**kwargs)(func)
return custom_min_far_option
def figsize_option(dflt='4,3', **kwargs): def figsize_option(dflt='4,3', **kwargs):
"""Get option for matplotlib figsize """Get option for matplotlib figsize
...@@ -362,7 +386,7 @@ def legends_option(**kwargs): ...@@ -362,7 +386,7 @@ def legends_option(**kwargs):
ctx.meta['legends'] = value ctx.meta['legends'] = value
return value return value
return click.option( return click.option(
'-l', '--legends', type=click.STRING, default=None, '-Z', '--legends', type=click.STRING, default=None,
help='The title for each system comma separated. ' help='The title for each system comma separated. '
'Example: --legends ISV,CNN', 'Example: --legends ISV,CNN',
callback=callback, **kwargs)(func) callback=callback, **kwargs)(func)
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
from __future__ import division, print_function from __future__ import division, print_function
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import math
import sys import sys
import os.path import os.path
import click import click
...@@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages ...@@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate from tabulate import tabulate
from .. import (far_threshold, plot, utils, ppndf) from .. import (far_threshold, plot, utils, ppndf)
LINESTYLES = [
(0, ()), #solid
(0, (4, 4)), #dashed
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (5, 1)), #densely dashed
(0, (1, 1)), #densely dotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (1, 10)) #loosely dotted
]
class MeasureBase(object): class MeasureBase(object):
"""Base class for metrics and plots. """Base class for metrics and plots.
This abstract class define the framework to plot or compute metrics from a This abstract class define the framework to plot or compute metrics from a
...@@ -77,28 +62,36 @@ class MeasureBase(object): ...@@ -77,28 +62,36 @@ class MeasureBase(object):
systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process` systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
(after the loop). (after the loop).
""" """
#init matplotlib, log files, ... # init matplotlib, log files, ...
self.init_process() self.init_process()
#iterates through the different systems and feed `compute` # iterates through the different systems and feed `compute`
#with the dev (and eval) scores of each system # with the dev (and eval) scores of each system
# Note that more than one dev or eval scores score can be passed to # Note that more than one dev or eval scores score can be passed to
# each system # each system
for idx in range(self.n_systems): for idx in range(self.n_systems):
# load scores for each system: get the corresponding arrays and
# base-name of files
input_scores, input_names = self._load_files( input_scores, input_names = self._load_files(
# Scores are given as followed:
# SysA-dev SysA-eval ... SysA-XX SysB-dev SysB-eval ... SysB-XX
# ------------------------------ ------------------------------
# First set of `self._min_arg` Second set of input files
# input files starting at for SysB
# index idx * self._min_arg
self._scores[idx * self._min_arg:(idx + 1) * self._min_arg] self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
) )
self.compute(idx, input_scores, input_names) self.compute(idx, input_scores, input_names)
#setup final configuration, plotting properties, ... # setup final configuration, plotting properties, ...
self.end_process() self.end_process()
#protected functions that need to be overwritten # protected functions that need to be overwritten
def init_process(self): def init_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
before iterating through the different systems. before iterating through the different systems.
Should reimplemented in derived classes""" Should reimplemented in derived classes"""
pass pass
#Main computations are done here in the subclasses # Main computations are done here in the subclasses
@abstractmethod @abstractmethod
def compute(self, idx, input_scores, input_names): def compute(self, idx, input_scores, input_names):
"""Compute metrics or plots from the given scores provided by """Compute metrics or plots from the given scores provided by
...@@ -116,7 +109,7 @@ class MeasureBase(object): ...@@ -116,7 +109,7 @@ class MeasureBase(object):
""" """
pass pass
#Things to do after the main iterative computations are done # Things to do after the main iterative computations are done
@abstractmethod @abstractmethod
def end_process(self): def end_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
...@@ -124,7 +117,7 @@ class MeasureBase(object): ...@@ -124,7 +117,7 @@ class MeasureBase(object):
Should reimplemented in derived classes""" Should reimplemented in derived classes"""
pass pass
#common protected functions # common protected functions
def _load_files(self, filepaths): def _load_files(self, filepaths):
''' Load the input files and return the base names of the files ''' Load the input files and return the base names of the files
...@@ -274,6 +267,12 @@ class PlotBase(MeasureBase): ...@@ -274,6 +267,12 @@ class PlotBase(MeasureBase):
self._points = 100 if 'points' not in ctx.meta else ctx.meta['points'] self._points = 100 if 'points' not in ctx.meta else ctx.meta['points']
self._split = None if 'split' not in ctx.meta else ctx.meta['split'] self._split = None if 'split' not in ctx.meta else ctx.meta['split']
self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim'] self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._min_dig = None
if 'min_far_value' in ctx.meta:
self._min_dig = int(math.log10(ctx.meta['min_far_value']))
elif self._axlim is not None:
self._min_dig = int(math.log10(self._axlim[0])
if self._axlim[0] != 0 else 0)
self._clayout = None if 'clayout' not in ctx.meta else\ self._clayout = None if 'clayout' not in ctx.meta else\
ctx.meta['clayout'] ctx.meta['clayout']
self._far_at = None if 'lines_at' not in ctx.meta else\ self._far_at = None if 'lines_at' not in ctx.meta else\
...@@ -290,6 +289,9 @@ class PlotBase(MeasureBase): ...@@ -290,6 +289,9 @@ class PlotBase(MeasureBase):
mpl.style.use(ctx.meta['style']) mpl.style.use(ctx.meta['style'])
self._nb_figs = 2 if self._eval and self._split else 1 self._nb_figs = 2 if self._eval and self._split else 1
self._colors = utils.get_colors(self.n_systems) self._colors = utils.get_colors(self.n_systems)
self._line_linestyles = False if 'line_linestyles' not in ctx.meta else \
ctx.meta['line_linestyles']
self._linestyles = utils.get_linestyles(self.n_systems, self._line_linestyles)
self._states = ['Development', 'Evaluation'] self._states = ['Development', 'Evaluation']
self._title = None if 'title' not in ctx.meta else ctx.meta['title'] self._title = None if 'title' not in ctx.meta else ctx.meta['title']
self._x_label = None if 'x_label' not in ctx.meta else\ self._x_label = None if 'x_label' not in ctx.meta else\
...@@ -383,7 +385,10 @@ class Roc(PlotBase): ...@@ -383,7 +385,10 @@ class Roc(PlotBase):
self._y_label = self._y_label or "1 - False Negative Rate" self._y_label = self._y_label or "1 - False Negative Rate"
#custom defaults #custom defaults
if self._axlim is None: if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0] self._axlim = [1e-4, 1.0, 0, 1.0]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig)
def compute(self, idx, input_scores, input_names): def compute(self, idx, input_scores, input_names):
''' Plot ROC for dev and eval data using ''' Plot ROC for dev and eval data using
...@@ -397,20 +402,20 @@ class Roc(PlotBase): ...@@ -397,20 +402,20 @@ class Roc(PlotBase):
mpl.figure(1) mpl.figure(1)
if self._eval: if self._eval:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.roc_for_far( plot.roc_for_far(
dev_neg, dev_pos, dev_neg, dev_pos,
color=self._colors[idx], linestyle=linestyle, far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx) label=self._label('development', dev_file, idx)
) )
linestyle = '--'
if self._split: if self._split:
mpl.figure(2) mpl.figure(2)
linestyle = LINESTYLES[idx % 14]
linestyle = '--' if not self._split else self._linestyles[idx]
plot.roc_for_far( plot.roc_for_far(
eval_neg, eval_pos, eval_neg, eval_pos, linestyle=linestyle,
color=self._colors[idx], linestyle=linestyle, far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx],
label=self._label('eval', eval_file, idx) label=self._label('eval', eval_file, idx)
) )
if self._far_at is not None: if self._far_at is not None:
...@@ -424,7 +429,8 @@ class Roc(PlotBase): ...@@ -424,7 +429,8 @@ class Roc(PlotBase):
else: else:
plot.roc_for_far( plot.roc_for_far(
dev_neg, dev_pos, dev_neg, dev_pos,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14], far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx) label=self._label('development', dev_file, idx)
) )
...@@ -441,6 +447,12 @@ class Det(PlotBase): ...@@ -441,6 +447,12 @@ class Det(PlotBase):
if self._x_rotation is None: if self._x_rotation is None:
self._x_rotation = 50 self._x_rotation = 50
if self._axlim is None:
self._axlim = [0.01, 99, 0.01, 99]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig) * 100
def compute(self, idx, input_scores, input_names): def compute(self, idx, input_scores, input_names):
''' Plot DET for dev and eval data using ''' Plot DET for dev and eval data using
:py:func:`bob.measure.plot.det`''' :py:func:`bob.measure.plot.det`'''
...@@ -453,15 +465,14 @@ class Det(PlotBase): ...@@ -453,15 +465,14 @@ class Det(PlotBase):
mpl.figure(1) mpl.figure(1)
if self._eval and eval_neg is not None: if self._eval and eval_neg is not None:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.det( plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx], dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=linestyle, linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx) label=self._label('development', dev_file, idx)
) )
if self._split: if self._split:
mpl.figure(2) mpl.figure(2)
linestyle = '--' if not self._split else LINESTYLES[idx % 14] linestyle = '--' if not self._split else self._linestyles[idx]
plot.det( plot.det(
eval_neg, eval_pos, self._points, color=self._colors[idx], eval_neg, eval_pos, self._points, color=self._colors[idx],
linestyle=linestyle, linestyle=linestyle,
...@@ -478,15 +489,12 @@ class Det(PlotBase): ...@@ -478,15 +489,12 @@ class Det(PlotBase):
else: else:
plot.det( plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx], dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=LINESTYLES[idx % 14], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx) label=self._label('development', dev_file, idx)
) )
def _set_axis(self): def _set_axis(self):
if self._axlim is not None and None not in self._axlim: plot.det_axis(self._axlim)
plot.det_axis(self._axlim)
else:
plot.det_axis([0.01, 99, 0.01, 99])
class Epc(PlotBase): class Epc(PlotBase):
''' Handles the plotting of EPC ''' ''' Handles the plotting of EPC '''
...@@ -513,7 +521,7 @@ class Epc(PlotBase): ...@@ -513,7 +521,7 @@ class Epc(PlotBase):
plot.epc( plot.epc(
dev_neg, dev_pos, eval_neg, eval_pos, self._points, dev_neg, dev_pos, eval_neg, eval_pos, self._points,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14], color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label( label=self._label(
'curve', dev_file + "_" + eval_file, idx 'curve', dev_file + "_" + eval_file, idx
) )
......
...@@ -30,7 +30,7 @@ def test_metrics(): ...@@ -30,7 +30,7 @@ def test_metrics():
assert result.exit_code == 0 assert result.exit_code == 0
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke( result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-l', commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-Z',
'A,B'] 'A,B']
) )
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output)
......
...@@ -137,6 +137,41 @@ def get_colors(n): ...@@ -137,6 +137,41 @@ def get_colors(n):
return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9'] return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9']
def get_linestyles(n, on=True):
"""Get a list of matplotlib linestyles
Parameters
----------
n : :obj:`int`
Number of linestyles to output
Returns
-------
:any:`list`
list of linestyles
"""
if not on:
return [None] * n
list_linestyles = [
(0, ()), #solid
(0, (1, 1)), #densely dotted
(0, (5, 5)), #dashed
(0, (5, 1)), #densely dashed
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (1, 10)) #loosely dotted
]
while n > len(list_linestyles):
list_linestyles += list_linestyles
return list_linestyles
def confidence_for_indicator_variable(x, n, alpha=0.05): def confidence_for_indicator_variable(x, n, alpha=0.05):
'''Calculates the confidence interval for proportion estimates '''Calculates the confidence interval for proportion estimates
The Clopper-Pearson interval method is used for estimating the confidence The Clopper-Pearson interval method is used for estimating the confidence
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment