Skip to content
Snippets Groups Projects
Commit 40a7e636 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'bugfix' into 'master'

Bugfix

See merge request !60
parents 83f4f98a 1c6e5bca
No related branches found
No related tags found
1 merge request!60Bugfix
Pipeline #
......@@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs):
# these are some constants required in this method
desiredTicks = [
"0.000001", "0.000002", "0.000005",
"0.00001", "0.00002", "0.00005",
"0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005",
......@@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs):
]
desiredLabels = [
"0.0001", "0.0002", "0.0005",
"0.001", "0.002", "0.005",
"0.01", "0.02", "0.05",
"0.1", "0.2", "0.5",
......
......@@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.eval_option()
@common_options.points_curve_option()
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.min_far_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
......@@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, evaluation, **kwargs):
......@@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.min_far_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
......@@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, evaluation, **kwargs):
......@@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs):
@common_options.y_label_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
......@@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs):
@common_options.legends_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
......@@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, evaluation, **kwargs):
......
......@@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs):
dflt
)
def linestyles_option(dflt=False, **kwargs):
''' Get option flag to turn on/off linestyles'''
return bool_option('line-linestyles', 'S', 'If given, applies a different '
'linestyles to each line.', dflt, **kwargs)
def cmc_option(**kwargs):
'''Get option flag to say if cmc scores'''
return bool_option('cmc', 'C', 'If set, CMC score files are provided')
return bool_option('cmc', 'C', 'If set, CMC score files are provided',
**kwargs)
def semilogx_option(dflt=False, **kwargs):
'''Option to use semilog X-axis'''
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt)
return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt,
**kwargs)
def show_dev_option(dflt=False, **kwargs):
'''Option to tell if should show dev histo'''
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt)
return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt,
**kwargs)
def print_filenames_option(dflt=True, **kwargs):
'''Option to tell if filenames should be in the title'''
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt)
return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt,
**kwargs)
def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
......@@ -278,6 +287,21 @@ def far_option(**kwargs):
callback=callback, show_default=True,**kwargs)(func)
return custom_far_option
def min_far_option(dflt=1e-4, **kwargs):
'''Get option to get min far value'''
def custom_min_far_option(func):
def callback(ctx, param, value):
if value is not None and (value > 1 or value < 0):
raise click.BadParameter("FAR value should be between 0 and 1")
ctx.meta['min_far_value'] = value
return value
return click.option(
'-M', '--min-far-value', type=click.FLOAT, default=dflt,
help='Select the minimum FAR value used in ROC and DET plots; '
'should be a power of 10.',
callback=callback, show_default=True,**kwargs)(func)
return custom_min_far_option
def figsize_option(dflt='4,3', **kwargs):
"""Get option for matplotlib figsize
......@@ -362,7 +386,7 @@ def legends_option(**kwargs):
ctx.meta['legends'] = value
return value
return click.option(
'-ls', '--legends', type=click.STRING, default=None,
'-Z', '--legends', type=click.STRING, default=None,
help='The title for each system comma separated. '
'Example: --legends ISV,CNN',
callback=callback, **kwargs)(func)
......
......@@ -2,6 +2,7 @@
from __future__ import division, print_function
from abc import ABCMeta, abstractmethod
import math
import sys
import os.path
import click
......@@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate
from .. import (far_threshold, plot, utils, ppndf)
LINESTYLES = [
(0, ()), #solid
(0, (4, 4)), #dashed
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (5, 1)), #densely dashed
(0, (1, 1)), #densely dotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (1, 10)) #loosely dotted
]
class MeasureBase(object):
"""Base class for metrics and plots.
This abstract class define the framework to plot or compute metrics from a
......@@ -77,28 +62,36 @@ class MeasureBase(object):
systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
(after the loop).
"""
#init matplotlib, log files, ...
# init matplotlib, log files, ...
self.init_process()
#iterates through the different systems and feed `compute`
#with the dev (and eval) scores of each system
# iterates through the different systems and feed `compute`
# with the dev (and eval) scores of each system
# Note that more than one dev or eval scores score can be passed to
# each system
for idx in range(self.n_systems):
# load scores for each system: get the corresponding arrays and
# base-name of files
input_scores, input_names = self._load_files(
# Scores are given as followed:
# SysA-dev SysA-eval ... SysA-XX SysB-dev SysB-eval ... SysB-XX
# ------------------------------ ------------------------------
# First set of `self._min_arg` Second set of input files
# input files starting at for SysB
# index idx * self._min_arg
self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
)
self.compute(idx, input_scores, input_names)
#setup final configuration, plotting properties, ...
# setup final configuration, plotting properties, ...
self.end_process()
#protected functions that need to be overwritten
# protected functions that need to be overwritten
def init_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
before iterating through the different systems.
Should reimplemented in derived classes"""
pass
#Main computations are done here in the subclasses
# Main computations are done here in the subclasses
@abstractmethod
def compute(self, idx, input_scores, input_names):
"""Compute metrics or plots from the given scores provided by
......@@ -116,7 +109,7 @@ class MeasureBase(object):
"""
pass
#Things to do after the main iterative computations are done
# Things to do after the main iterative computations are done
@abstractmethod
def end_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
......@@ -124,7 +117,7 @@ class MeasureBase(object):
Should reimplemented in derived classes"""
pass
#common protected functions
# common protected functions
def _load_files(self, filepaths):
''' Load the input files and return the base names of the files
......@@ -274,6 +267,12 @@ class PlotBase(MeasureBase):
self._points = 100 if 'points' not in ctx.meta else ctx.meta['points']
self._split = None if 'split' not in ctx.meta else ctx.meta['split']
self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
self._min_dig = None
if 'min_far_value' in ctx.meta:
self._min_dig = int(math.log10(ctx.meta['min_far_value']))
elif self._axlim is not None:
self._min_dig = int(math.log10(self._axlim[0])
if self._axlim[0] != 0 else 0)
self._clayout = None if 'clayout' not in ctx.meta else\
ctx.meta['clayout']
self._far_at = None if 'lines_at' not in ctx.meta else\
......@@ -290,6 +289,9 @@ class PlotBase(MeasureBase):
mpl.style.use(ctx.meta['style'])
self._nb_figs = 2 if self._eval and self._split else 1
self._colors = utils.get_colors(self.n_systems)
self._line_linestyles = False if 'line_linestyles' not in ctx.meta else \
ctx.meta['line_linestyles']
self._linestyles = utils.get_linestyles(self.n_systems, self._line_linestyles)
self._states = ['Development', 'Evaluation']
self._title = None if 'title' not in ctx.meta else ctx.meta['title']
self._x_label = None if 'x_label' not in ctx.meta else\
......@@ -383,7 +385,10 @@ class Roc(PlotBase):
self._y_label = self._y_label or "1 - False Negative Rate"
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
self._axlim = [1e-4, 1.0, 0, 1.0]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig)
def compute(self, idx, input_scores, input_names):
''' Plot ROC for dev and eval data using
......@@ -397,20 +402,20 @@ class Roc(PlotBase):
mpl.figure(1)
if self._eval:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.roc_for_far(
dev_neg, dev_pos,
color=self._colors[idx], linestyle=linestyle,
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
linestyle = '--'
if self._split:
mpl.figure(2)
linestyle = LINESTYLES[idx % 14]
linestyle = '--' if not self._split else self._linestyles[idx]
plot.roc_for_far(
eval_neg, eval_pos,
color=self._colors[idx], linestyle=linestyle,
eval_neg, eval_pos, linestyle=linestyle,
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx],
label=self._label('eval', eval_file, idx)
)
if self._far_at is not None:
......@@ -424,7 +429,8 @@ class Roc(PlotBase):
else:
plot.roc_for_far(
dev_neg, dev_pos,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
far_values=plot.log_values(self._min_dig or -4),
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
......@@ -441,6 +447,12 @@ class Det(PlotBase):
if self._x_rotation is None:
self._x_rotation = 50
if self._axlim is None:
self._axlim = [0.01, 99, 0.01, 99]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig) * 100
def compute(self, idx, input_scores, input_names):
''' Plot DET for dev and eval data using
:py:func:`bob.measure.plot.det`'''
......@@ -453,15 +465,14 @@ class Det(PlotBase):
mpl.figure(1)
if self._eval and eval_neg is not None:
linestyle = '-' if not self._split else LINESTYLES[idx % 14]
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=linestyle,
linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
if self._split:
mpl.figure(2)
linestyle = '--' if not self._split else LINESTYLES[idx % 14]
linestyle = '--' if not self._split else self._linestyles[idx]
plot.det(
eval_neg, eval_pos, self._points, color=self._colors[idx],
linestyle=linestyle,
......@@ -478,15 +489,12 @@ class Det(PlotBase):
else:
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=LINESTYLES[idx % 14],
linestyle=self._linestyles[idx],
label=self._label('development', dev_file, idx)
)
def _set_axis(self):
if self._axlim is not None and None not in self._axlim:
plot.det_axis(self._axlim)
else:
plot.det_axis([0.01, 99, 0.01, 99])
plot.det_axis(self._axlim)
class Epc(PlotBase):
''' Handles the plotting of EPC '''
......@@ -513,7 +521,7 @@ class Epc(PlotBase):
plot.epc(
dev_neg, dev_pos, eval_neg, eval_pos, self._points,
color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label(
'curve', dev_file + "_" + eval_file, idx
)
......
......@@ -30,7 +30,7 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-ls',
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-Z',
'A,B']
)
assert result.exit_code == 0, (result.exit_code, result.output)
......
......@@ -137,6 +137,41 @@ def get_colors(n):
return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9']
def get_linestyles(n, on=True):
"""Get a list of matplotlib linestyles
Parameters
----------
n : :obj:`int`
Number of linestyles to output
Returns
-------
:any:`list`
list of linestyles
"""
if not on:
return [None] * n
list_linestyles = [
(0, ()), #solid
(0, (1, 1)), #densely dotted
(0, (5, 5)), #dashed
(0, (5, 1)), #densely dashed
(0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted
(0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
(0, (3, 5, 1, 5, 1, 5)), #dashdotdotted
(0, (3, 1, 1, 1)), #densely dashdotted
(0, (1, 5)), #dotted
(0, (3, 5, 1, 5)), #dashdotted
(0, (5, 10)), #loosely dashed
(0, (3, 10, 1, 10)), #loosely dashdotted
(0, (1, 10)) #loosely dotted
]
while n > len(list_linestyles):
list_linestyles += list_linestyles
return list_linestyles
def confidence_for_indicator_variable(x, n, alpha=0.05):
'''Calculates the confidence interval for proportion estimates
The Clopper-Pearson interval method is used for estimating the confidence
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment