diff --git a/bob/measure/plot.py b/bob/measure/plot.py index 536032b0949b9868967ae92c5918757d82c3cd3b..365ca3332819abdf9b52aa904958900676869d85 100644 --- a/bob/measure/plot.py +++ b/bob/measure/plot.py @@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs): # these are some constants required in this method desiredTicks = [ + "0.000001", "0.000002", "0.000005", "0.00001", "0.00002", "0.00005", "0.0001", "0.0002", "0.0005", "0.001", "0.002", "0.005", @@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs): ] desiredLabels = [ + "0.0001", "0.0002", "0.0005", "0.001", "0.002", "0.005", "0.01", "0.02", "0.05", "0.1", "0.2", "0.5", diff --git a/bob/measure/script/commands.py b/bob/measure/script/commands.py index e42704ee2975253cbcf5cc96c7a2673360082dcf..bbabf61ec638c450df58c83636501082de328606 100644 --- a/bob/measure/script/commands.py +++ b/bob/measure/script/commands.py @@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs): @common_options.eval_option() @common_options.points_curve_option() @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1]) +@common_options.min_far_option() @common_options.x_rotation_option() @common_options.x_label_option() @common_options.y_label_option() @@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs): @common_options.const_layout_option() @common_options.figsize_option() @common_options.style_option() +@common_options.linestyles_option() @verbosity_option() @click.pass_context def roc(ctx, scores, evaluation, **kwargs): @@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs): @common_options.sep_dev_eval_option() @common_options.eval_option() @common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95]) +@common_options.min_far_option() @common_options.x_rotation_option(dflt=45) @common_options.x_label_option() @common_options.y_label_option() @@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs): @common_options.const_layout_option() @common_options.figsize_option() @common_options.style_option() +@common_options.linestyles_option() @verbosity_option() @click.pass_context def det(ctx, scores, evaluation, **kwargs): @@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs): @common_options.y_label_option() @common_options.figsize_option() @common_options.style_option() +@common_options.linestyles_option() @verbosity_option() @click.pass_context def epc(ctx, scores, **kwargs): @@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs): @common_options.legends_option() @common_options.figsize_option() @common_options.style_option() +@common_options.linestyles_option() @verbosity_option() @click.pass_context def hist(ctx, scores, evaluation, **kwargs): @@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs): @common_options.const_layout_option() @common_options.figsize_option() @common_options.style_option() +@common_options.linestyles_option() @verbosity_option() @click.pass_context def evaluate(ctx, scores, evaluation, **kwargs): diff --git a/bob/measure/script/common_options.py b/bob/measure/script/common_options.py index b26c677e25bf64bacf42d99d80e2f61d6ac89823..df150985df387d3c2ba43701992df53daa6dacc9 100644 --- a/bob/measure/script/common_options.py +++ b/bob/measure/script/common_options.py @@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs): dflt ) +def linestyles_option(dflt=False, **kwargs): + ''' Get option flag to turn on/off linestyles''' + return bool_option('line-linestyles', 'S', 'If given, applies a different ' + 'linestyles to each line.', dflt, **kwargs) + def cmc_option(**kwargs): '''Get option flag to say if cmc scores''' - return bool_option('cmc', 'C', 'If set, CMC score files are provided') + return bool_option('cmc', 'C', 'If set, CMC score files are provided', + **kwargs) def semilogx_option(dflt=False, **kwargs): '''Option to use semilog X-axis''' - return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt) + return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt, + **kwargs) def show_dev_option(dflt=False, **kwargs): '''Option to tell if should show dev histo''' - return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt) + return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt, + **kwargs) def print_filenames_option(dflt=True, **kwargs): '''Option to tell if filenames should be in the title''' - return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt) + return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt, + **kwargs) def const_layout_option(dflt=True, **kwargs): '''Option to set matplotlib constrained_layout''' @@ -278,6 +287,21 @@ def far_option(**kwargs): callback=callback, show_default=True,**kwargs)(func) return custom_far_option +def min_far_option(dflt=1e-4, **kwargs): + '''Get option to get min far value''' + def custom_min_far_option(func): + def callback(ctx, param, value): + if value is not None and (value > 1 or value < 0): + raise click.BadParameter("FAR value should be between 0 and 1") + ctx.meta['min_far_value'] = value + return value + return click.option( + '-M', '--min-far-value', type=click.FLOAT, default=dflt, + help='Select the minimum FAR value used in ROC and DET plots; ' + 'should be a power of 10.', + callback=callback, show_default=True,**kwargs)(func) + return custom_min_far_option + def figsize_option(dflt='4,3', **kwargs): """Get option for matplotlib figsize @@ -362,7 +386,7 @@ def legends_option(**kwargs): ctx.meta['legends'] = value return value return click.option( - '-l', '--legends', type=click.STRING, default=None, + '-Z', '--legends', type=click.STRING, default=None, help='The title for each system comma separated. ' 'Example: --legends ISV,CNN', callback=callback, **kwargs)(func) diff --git a/bob/measure/script/figure.py b/bob/measure/script/figure.py index 60c5746c9535da7f98706dd420f344b461b7d410..95c2c16a91ae5d44966b27d32bd2cb4fadd7e9ff 100644 --- a/bob/measure/script/figure.py +++ b/bob/measure/script/figure.py @@ -2,6 +2,7 @@ from __future__ import division, print_function from abc import ABCMeta, abstractmethod +import math import sys import os.path import click @@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages from tabulate import tabulate from .. import (far_threshold, plot, utils, ppndf) -LINESTYLES = [ - (0, ()), #solid - (0, (4, 4)), #dashed - (0, (1, 5)), #dotted - (0, (3, 5, 1, 5)), #dashdotted - (0, (3, 5, 1, 5, 1, 5)), #dashdotdotted - (0, (5, 1)), #densely dashed - (0, (1, 1)), #densely dotted - (0, (3, 1, 1, 1)), #densely dashdotted - (0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted - (0, (5, 10)), #loosely dashed - (0, (3, 10, 1, 10)), #loosely dashdotted - (0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted - (0, (1, 10)) #loosely dotted -] - class MeasureBase(object): """Base class for metrics and plots. This abstract class define the framework to plot or compute metrics from a @@ -77,28 +62,36 @@ class MeasureBase(object): systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process` (after the loop). """ - #init matplotlib, log files, ... + # init matplotlib, log files, ... self.init_process() - #iterates through the different systems and feed `compute` - #with the dev (and eval) scores of each system + # iterates through the different systems and feed `compute` + # with the dev (and eval) scores of each system # Note that more than one dev or eval scores score can be passed to # each system for idx in range(self.n_systems): + # load scores for each system: get the corresponding arrays and + # base-name of files input_scores, input_names = self._load_files( + # Scores are given as followed: + # SysA-dev SysA-eval ... SysA-XX SysB-dev SysB-eval ... SysB-XX + # ------------------------------ ------------------------------ + # First set of `self._min_arg` Second set of input files + # input files starting at for SysB + # index idx * self._min_arg self._scores[idx * self._min_arg:(idx + 1) * self._min_arg] ) self.compute(idx, input_scores, input_names) - #setup final configuration, plotting properties, ... + # setup final configuration, plotting properties, ... self.end_process() - #protected functions that need to be overwritten + # protected functions that need to be overwritten def init_process(self): """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run before iterating through the different systems. Should reimplemented in derived classes""" pass - #Main computations are done here in the subclasses + # Main computations are done here in the subclasses @abstractmethod def compute(self, idx, input_scores, input_names): """Compute metrics or plots from the given scores provided by @@ -116,7 +109,7 @@ class MeasureBase(object): """ pass - #Things to do after the main iterative computations are done + # Things to do after the main iterative computations are done @abstractmethod def end_process(self): """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run @@ -124,7 +117,7 @@ class MeasureBase(object): Should reimplemented in derived classes""" pass - #common protected functions + # common protected functions def _load_files(self, filepaths): ''' Load the input files and return the base names of the files @@ -274,6 +267,12 @@ class PlotBase(MeasureBase): self._points = 100 if 'points' not in ctx.meta else ctx.meta['points'] self._split = None if 'split' not in ctx.meta else ctx.meta['split'] self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim'] + self._min_dig = None + if 'min_far_value' in ctx.meta: + self._min_dig = int(math.log10(ctx.meta['min_far_value'])) + elif self._axlim is not None: + self._min_dig = int(math.log10(self._axlim[0]) + if self._axlim[0] != 0 else 0) self._clayout = None if 'clayout' not in ctx.meta else\ ctx.meta['clayout'] self._far_at = None if 'lines_at' not in ctx.meta else\ @@ -290,6 +289,9 @@ class PlotBase(MeasureBase): mpl.style.use(ctx.meta['style']) self._nb_figs = 2 if self._eval and self._split else 1 self._colors = utils.get_colors(self.n_systems) + self._line_linestyles = False if 'line_linestyles' not in ctx.meta else \ + ctx.meta['line_linestyles'] + self._linestyles = utils.get_linestyles(self.n_systems, self._line_linestyles) self._states = ['Development', 'Evaluation'] self._title = None if 'title' not in ctx.meta else ctx.meta['title'] self._x_label = None if 'x_label' not in ctx.meta else\ @@ -383,7 +385,10 @@ class Roc(PlotBase): self._y_label = self._y_label or "1 - False Negative Rate" #custom defaults if self._axlim is None: - self._axlim = [1e-4, 1.0, 1e-4, 1.0] + self._axlim = [1e-4, 1.0, 0, 1.0] + + if self._min_dig is not None: + self._axlim[0] = math.pow(10, self._min_dig) def compute(self, idx, input_scores, input_names): ''' Plot ROC for dev and eval data using @@ -397,20 +402,20 @@ class Roc(PlotBase): mpl.figure(1) if self._eval: - linestyle = '-' if not self._split else LINESTYLES[idx % 14] plot.roc_for_far( dev_neg, dev_pos, - color=self._colors[idx], linestyle=linestyle, + far_values=plot.log_values(self._min_dig or -4), + color=self._colors[idx], linestyle=self._linestyles[idx], label=self._label('development', dev_file, idx) ) - linestyle = '--' if self._split: mpl.figure(2) - linestyle = LINESTYLES[idx % 14] + linestyle = '--' if not self._split else self._linestyles[idx] plot.roc_for_far( - eval_neg, eval_pos, - color=self._colors[idx], linestyle=linestyle, + eval_neg, eval_pos, linestyle=linestyle, + far_values=plot.log_values(self._min_dig or -4), + color=self._colors[idx], label=self._label('eval', eval_file, idx) ) if self._far_at is not None: @@ -424,7 +429,8 @@ class Roc(PlotBase): else: plot.roc_for_far( dev_neg, dev_pos, - color=self._colors[idx], linestyle=LINESTYLES[idx % 14], + far_values=plot.log_values(self._min_dig or -4), + color=self._colors[idx], linestyle=self._linestyles[idx], label=self._label('development', dev_file, idx) ) @@ -441,6 +447,12 @@ class Det(PlotBase): if self._x_rotation is None: self._x_rotation = 50 + if self._axlim is None: + self._axlim = [0.01, 99, 0.01, 99] + + if self._min_dig is not None: + self._axlim[0] = math.pow(10, self._min_dig) * 100 + def compute(self, idx, input_scores, input_names): ''' Plot DET for dev and eval data using :py:func:`bob.measure.plot.det`''' @@ -453,15 +465,14 @@ class Det(PlotBase): mpl.figure(1) if self._eval and eval_neg is not None: - linestyle = '-' if not self._split else LINESTYLES[idx % 14] plot.det( dev_neg, dev_pos, self._points, color=self._colors[idx], - linestyle=linestyle, + linestyle=self._linestyles[idx], label=self._label('development', dev_file, idx) ) if self._split: mpl.figure(2) - linestyle = '--' if not self._split else LINESTYLES[idx % 14] + linestyle = '--' if not self._split else self._linestyles[idx] plot.det( eval_neg, eval_pos, self._points, color=self._colors[idx], linestyle=linestyle, @@ -478,15 +489,12 @@ class Det(PlotBase): else: plot.det( dev_neg, dev_pos, self._points, color=self._colors[idx], - linestyle=LINESTYLES[idx % 14], + linestyle=self._linestyles[idx], label=self._label('development', dev_file, idx) ) def _set_axis(self): - if self._axlim is not None and None not in self._axlim: - plot.det_axis(self._axlim) - else: - plot.det_axis([0.01, 99, 0.01, 99]) + plot.det_axis(self._axlim) class Epc(PlotBase): ''' Handles the plotting of EPC ''' @@ -513,7 +521,7 @@ class Epc(PlotBase): plot.epc( dev_neg, dev_pos, eval_neg, eval_pos, self._points, - color=self._colors[idx], linestyle=LINESTYLES[idx % 14], + color=self._colors[idx], linestyle=self._linestyles[idx], label=self._label( 'curve', dev_file + "_" + eval_file, idx ) diff --git a/bob/measure/test_script.py b/bob/measure/test_script.py index 178a30a8278466a0eae12b73f8846fe460f46560..e885d55448050bc61f07af17584e63363a7910ae 100644 --- a/bob/measure/test_script.py +++ b/bob/measure/test_script.py @@ -30,7 +30,7 @@ def test_metrics(): assert result.exit_code == 0 with runner.isolated_filesystem(): result = runner.invoke( - commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-l', + commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-Z', 'A,B'] ) assert result.exit_code == 0, (result.exit_code, result.output) diff --git a/bob/measure/utils.py b/bob/measure/utils.py index ef399e33db9c79221b37b54540025955500e5fd6..15ff8a83f7306576b256525ff15ee129c37fdb52 100644 --- a/bob/measure/utils.py +++ b/bob/measure/utils.py @@ -137,6 +137,41 @@ def get_colors(n): return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9'] +def get_linestyles(n, on=True): + """Get a list of matplotlib linestyles + + Parameters + ---------- + n : :obj:`int` + Number of linestyles to output + + Returns + ------- + :any:`list` + list of linestyles + """ + if not on: + return [None] * n + + list_linestyles = [ + (0, ()), #solid + (0, (1, 1)), #densely dotted + (0, (5, 5)), #dashed + (0, (5, 1)), #densely dashed + (0, (3, 1, 1, 1, 1, 1)), #densely dashdotdotted + (0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted + (0, (3, 5, 1, 5, 1, 5)), #dashdotdotted + (0, (3, 1, 1, 1)), #densely dashdotted + (0, (1, 5)), #dotted + (0, (3, 5, 1, 5)), #dashdotted + (0, (5, 10)), #loosely dashed + (0, (3, 10, 1, 10)), #loosely dashdotted + (0, (1, 10)) #loosely dotted + ] + while n > len(list_linestyles): + list_linestyles += list_linestyles + return list_linestyles + def confidence_for_indicator_variable(x, n, alpha=0.05): '''Calculates the confidence interval for proportion estimates The Clopper-Pearson interval method is used for estimating the confidence