Commit 1c6e5bca authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Add min-far-value option, rename -l (already used) to -Z, change the way...

Add min-far-value option, rename -l (already used) to -Z, change the way linestyles are handled and make use of them optional
parent 77f11de1
Pipeline #19647 passed with stage
in 40 minutes and 8 seconds
......@@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs):
# these are some constants required in this method
desiredTicks = [
"0.000001", "0.000002", "0.000005",
"0.00001", "0.00002", "0.00005",
"0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005",
......@@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs):
]
desiredLabels = [
"0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005",
"0.01", "0.02", "0.05",
"0.1", "0.2", "0.5",
......
......@@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.eval_option()
@common_options.points_curve_option()
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.min_far_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
......@@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, evaluation, **kwargs):
......@@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.min_far_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
......@@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, evaluation, **kwargs):
......@@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs):
@common_options.y_label_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
......@@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs):
@common_options.legends_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
......@@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, evaluation, **kwargs):
......
......@@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs):
dflt
)
def linestyles_option(dflt=False, **kwargs):
''' Get option flag to turn on/off linestyles'''
return bool_option('line-linestyles', 'S', 'If given, applies a different '
'linestyles to each line.', dflt, **kwargs)
def cmc_option(**kwargs):
'''Get option flag to say if cmc scores'''
return bool_option('cmc', 'C', 'If set, CMC score files are provided')
return bool_option('cmc', 'C', 'If set, CMC score files are provided',
**kwargs)
def semilogx_option(dflt=False, **kwargs):
'''Option to use semilog X-axis'''
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt)
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt,
**kwargs)
def show_dev_option(dflt=False, **kwargs):
'''Option to tell if should show dev histo'''
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt)
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt,
**kwargs)
def print_filenames_option(dflt=True, **kwargs):
'''Option to tell if filenames should be in the title'''
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt)
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt,
**kwargs)
def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
......@@ -278,6 +287,21 @@ def far_option(**kwargs):
callback=callback, show_default=True,**kwargs)(func)
return custom_far_option
def min_far_option(dflt=1e-4, **kwargs):
'''Get option to get min far value'''
def custom_min_far_option(func):
def callback(ctx, param, value):
if value is not None and (value > 1 or value < 0):
raise click.BadParameter("FAR value should be between 0 and 1")
ctx.meta['min_far_value'] = value
return value
return click.option(
'-M', '--min-far-value', type=click.FLOAT, default=dflt,
help='Select the minimum FAR value used in ROC and DET plots; '
'should be a power of 10.',
callback=callback, show_default=True,**kwargs)(func)
return custom_min_far_option
def figsize_option(dflt='4,3', **kwargs):
"""Get option for matplotlib figsize
......@@ -362,7 +386,7 @@ def legends_option(**kwargs):
ctx.meta['legends'] = value
return value
return click.option(
'-l', '--legends', type=click.STRING, default=None,
'-Z', '--legends', type=click.STRING, default=None,
help='The title for each system comma separated. '
'Example: --legends ISV,CNN',
callback=callback, **kwargs)(func)
......
......@@ -2,6 +2,7 @@
from __future__ import division, print_function
from abc import ABCMeta, abstractmethod
import math
import sys
import os.path
import click
......@@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate
from .. import (far_threshold, plot, utils, ppndf)
LINESTYLES = [
(0, ()), #solid
(0, (4, 4)), #dashed
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (5, 1)), #densely dashed
(0, (1, 1)), #densely dotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (1, 10)) #loosely dotted
]
class MeasureBase(object):
"""Base class for metrics and plots.
This abstract class define the framework to plot or compute metrics from a
......@@ -77,28 +62,36 @@ class MeasureBase(object):
systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
(after the loop).
"""
#init matplotlib, log files, ...
# init matplotlib, log files, ...
self.init_process()
#iterates through the different systems and feed `compute`
#with the dev (and eval) scores of each system
# iterates through the different systems and feed `compute`
# with the dev (and eval) scores of each system
# Note that more than one dev or eval scores score can be passed to
# each system
for idx in range(self.n_systems):
# load scores for each system: get the corresponding arrays and
# base-name of files
input_scores, input_names = self._load_files(
# Scores are given as followed:
# SysA-dev SysA-eval ... SysA-XX SysB-dev SysB-eval ... SysB-XX
# ------------------------------ ------------------------------
# First set of `self._min_arg` Second set of input files
# input files starting at for SysB
# index idx * self._min_arg
self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
)
self.compute(idx, input_scores, input_names)
#setup final configuration, plotting properties, ...
# setup final configuration, plotting properties, ...
self.end_process()
#protected functions that need to be overwritten
# protected functions that need to be overwritten
def init_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
before iterating through the different systems.
Should reimplemented in derived classes"""
pass
#Main computations are done here in the subclasses
# Main computations are done here in the subclasses
@abstractmethod
def compute(self, idx, input_scores, input_names):
"""Compute metrics or plots from the given scores provided by
......@@ -116,7 +109,7 @@ class MeasureBase(object):
"""
pass
#Things to do after the main iterative computations are done
# Things to do after the main iterative computations are done
@abstractmethod
def end_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
......@@ -124,7 +117,7 @@ class MeasureBase(object):
Should reimplemented in derived classes"""
pass
#common protected functions
# common protected functions
def _load_files(self, filepaths):
''' Load the input files and return the base names of the files
......@@ -274,6 +267,12 @@ class PlotBase(MeasureBase):
self._points = 100 if 'points' not in ctx.meta else ctx.meta['points']
self._split = None if 'split' not in ctx.meta else ctx.meta['split']
self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._min_dig = None
if 'min_far_value' in ctx.meta:
self._min_dig = int(math.log10(ctx.meta['min_far_value']))
elif self._axlim is not None:
self._min_dig = int(math.log10(self._axlim[0])
if self._axlim[0] != 0 else 0)
self._clayout = None if 'clayout' not in ctx.meta else\
ctx.meta['clayout']
self._far_at = None if 'lines_at' not in ctx.meta else\
......@@ -290,6 +289,9 @@ class PlotBase(MeasureBase):
mpl.style.use(ctx.meta['style'])
self._nb_figs = 2 if self._eval and self._split else 1
self._colors = utils.get_colors(self.n_systems)
self._line_linestyles = False if 'line_linestyles' not in ctx.meta else \
ctx.meta['line_linestyles']
self._linestyles = utils.get_linestyles(self.n_systems, self._line_linestyles)
self._states = ['Development', 'Evaluation']
self._title = None if 'title' not in ctx.meta else ctx.meta['title']
self._x_label = None if 'x_label' not in ctx.meta else\
......@@ -383,7 +385,10 @@ class Roc(PlotBase):
self._y_label = self._y_label or "1 - False Negative Rate"
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
self._axlim = [1e-4, 1.0, 0, 1.0]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig)
def compute(self, idx, input_scores, input_names):
''' Plot ROC for dev and eval data using
......@@ -397,20 +402,20 @@ class Roc(PlotBase):
mpl.figure(1)
if self._eval:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.roc_for_far(
dev_neg, dev_pos,
color=self._colors[idx], linestyle=linestyle,
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
linestyle = '--'
if self._split:
mpl.figure(2)
linestyle = LINESTYLES[idx % 14]
linestyle = '--' if not self._split else self._linestyles[idx]
plot.roc_for_far(
eval_neg, eval_pos,
color=self._colors[idx], linestyle=linestyle,
eval_neg, eval_pos, linestyle=linestyle,
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx],
label=self._label('eval', eval_file, idx)
)
if self._far_at is not None:
......@@ -424,7 +429,8 @@ class Roc(PlotBase):
else:
plot.roc_for_far(
dev_neg, dev_pos,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
......@@ -441,6 +447,12 @@ class Det(PlotBase):
if self._x_rotation is None:
self._x_rotation = 50
if self._axlim is None:
self._axlim = [0.01, 99, 0.01, 99]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig) * 100
def compute(self, idx, input_scores, input_names):
''' Plot DET for dev and eval data using
:py:func:`bob.measure.plot.det`'''
......@@ -453,15 +465,14 @@ class Det(PlotBase):
mpl.figure(1)
if self._eval and eval_neg is not None:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=linestyle,
linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
if self._split:
mpl.figure(2)
linestyle = '--' if not self._split else LINESTYLES[idx % 14]
linestyle = '--' if not self._split else self._linestyles[idx]
plot.det(
eval_neg, eval_pos, self._points, color=self._colors[idx],
linestyle=linestyle,
......@@ -478,15 +489,12 @@ class Det(PlotBase):
else:
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=LINESTYLES[idx % 14],
linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
def _set_axis(self):
if self._axlim is not None and None not in self._axlim:
plot.det_axis(self._axlim)
else:
plot.det_axis([0.01, 99, 0.01, 99])
plot.det_axis(self._axlim)
class Epc(PlotBase):
''' Handles the plotting of EPC '''
......@@ -513,7 +521,7 @@ class Epc(PlotBase):
plot.epc(
dev_neg, dev_pos, eval_neg, eval_pos, self._points,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label(
'curve', dev_file + "_" + eval_file, idx
)
......
......@@ -30,7 +30,7 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-l',
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-Z',
'A,B']
)
assert result.exit_code == 0, (result.exit_code, result.output)
......
......@@ -137,6 +137,41 @@ def get_colors(n):
return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9']
def get_linestyles(n, on=True):
"""Get a list of matplotlib linestyles
Parameters
----------
n : :obj:`int`
Number of linestyles to output
Returns
-------
:any:`list`
list of linestyles
"""
if not on:
return [None] * n
list_linestyles = [
(0, ()), #solid
(0, (1, 1)), #densely dotted
(0, (5, 5)), #dashed
(0, (5, 1)), #densely dashed
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (1, 10)) #loosely dotted
]
while n > len(list_linestyles):
list_linestyles += list_linestyles
return list_linestyles
def confidence_for_indicator_variable(x, n, alpha=0.05):
'''Calculates the confidence interval for proportion estimates
The Clopper-Pearson interval method is used for estimating the confidence
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment