Add min-far-value option, rename -l (already used) to -Z, change the way...

Add min-far-value option, rename -l (already used) to -Z, change the way linestyles are handled and make use of them optional

bob/measure/plot.py

@@ -346,6 +346,7 @@ def det(negatives, positives, npoints=100, **kwargs):
 
   # these are some constants required in this method
   desiredTicks = [
+      "0.000001", "0.000002", "0.000005",
       "0.00001", "0.00002", "0.00005",
       "0.0001", "0.0002", "0.0005",
       "0.001", "0.002", "0.005",
@@ -358,6 +359,7 @@ def det(negatives, positives, npoints=100, **kwargs):
   ]
 
   desiredLabels = [
+      "0.0001", "0.0002", "0.0005",
       "0.001", "0.002", "0.005",
       "0.01", "0.02", "0.05",
       "0.1", "0.2", "0.5",

bob/measure/script/commands.py

@@ -49,6 +49,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
 @common_options.eval_option()
 @common_options.points_curve_option()
 @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
+@common_options.min_far_option()
 @common_options.x_rotation_option()
 @common_options.x_label_option()
 @common_options.y_label_option()
@@ -56,6 +57,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
 @common_options.const_layout_option()
 @common_options.figsize_option()
 @common_options.style_option()
+@common_options.linestyles_option()
 @verbosity_option()
 @click.pass_context
 def roc(ctx, scores, evaluation, **kwargs):
@@ -87,6 +89,7 @@ def roc(ctx, scores, evaluation, **kwargs):
 @common_options.sep_dev_eval_option()
 @common_options.eval_option()
 @common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
+@common_options.min_far_option()
 @common_options.x_rotation_option(dflt=45)
 @common_options.x_label_option()
 @common_options.y_label_option()
@@ -95,6 +98,7 @@ def roc(ctx, scores, evaluation, **kwargs):
 @common_options.const_layout_option()
 @common_options.figsize_option()
 @common_options.style_option()
+@common_options.linestyles_option()
 @verbosity_option()
 @click.pass_context
 def det(ctx, scores, evaluation, **kwargs):
@@ -128,6 +132,7 @@ def det(ctx, scores, evaluation, **kwargs):
 @common_options.y_label_option()
 @common_options.figsize_option()
 @common_options.style_option()
+@common_options.linestyles_option()
 @verbosity_option()
 @click.pass_context
 def epc(ctx, scores, **kwargs):
@@ -161,6 +166,7 @@ def epc(ctx, scores, **kwargs):
 @common_options.legends_option()
 @common_options.figsize_option()
 @common_options.style_option()
+@common_options.linestyles_option()
 @verbosity_option()
 @click.pass_context
 def hist(ctx, scores, evaluation, **kwargs):
@@ -201,6 +207,7 @@ def hist(ctx, scores, evaluation, **kwargs):
 @common_options.const_layout_option()
 @common_options.figsize_option()
 @common_options.style_option()
+@common_options.linestyles_option()
 @verbosity_option()
 @click.pass_context
 def evaluate(ctx, scores, evaluation, **kwargs):

bob/measure/script/common_options.py

@@ -69,21 +69,30 @@ def sep_dev_eval_option(dflt=True, **kwargs):
         dflt
     )
 
+def linestyles_option(dflt=False, **kwargs):
+    ''' Get option flag to turn on/off linestyles'''
+    return bool_option('line-linestyles', 'S', 'If given, applies a different '
+                       'linestyles to each line.', dflt, **kwargs)
+
 def cmc_option(**kwargs):
     '''Get option flag to say if cmc scores'''
-    return bool_option('cmc', 'C', 'If set, CMC score files are provided')
+    return bool_option('cmc', 'C', 'If set, CMC score files are provided',
+                       **kwargs)
 
 def semilogx_option(dflt=False, **kwargs):
     '''Option to use semilog X-axis'''
-    return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt)
+    return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt,
+                       **kwargs)
 
 def show_dev_option(dflt=False, **kwargs):
     '''Option to tell if should show dev histo'''
-    return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt)
+    return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt,
+                       **kwargs)
 
 def print_filenames_option(dflt=True, **kwargs):
     '''Option to tell if filenames should be in the title'''
-    return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt)
+    return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt,
+                       **kwargs)
 
 def const_layout_option(dflt=True, **kwargs):
     '''Option to set matplotlib constrained_layout'''
@@ -278,6 +287,21 @@ def far_option(**kwargs):
             callback=callback, show_default=True,**kwargs)(func)
     return custom_far_option
 
+def min_far_option(dflt=1e-4, **kwargs):
+    '''Get option to get min far value'''
+    def custom_min_far_option(func):
+        def callback(ctx, param, value):
+            if value is not None and (value > 1 or value < 0):
+                raise click.BadParameter("FAR value should be between 0 and 1")
+            ctx.meta['min_far_value'] = value
+            return value
+        return click.option(
+            '-M', '--min-far-value', type=click.FLOAT, default=dflt,
+            help='Select the minimum FAR value used in ROC and DET plots; '
+            'should be a power of 10.',
+            callback=callback, show_default=True,**kwargs)(func)
+    return custom_min_far_option
+
 def figsize_option(dflt='4,3', **kwargs):
     """Get option for matplotlib figsize
 
@@ -362,7 +386,7 @@ def legends_option(**kwargs):
             ctx.meta['legends'] = value
             return value
         return click.option(
-            '-l', '--legends', type=click.STRING, default=None,
+            '-Z', '--legends', type=click.STRING, default=None,
             help='The title for each system comma separated. '
             'Example: --legends ISV,CNN',
             callback=callback, **kwargs)(func)

bob/measure/script/figure.py

@@ -2,6 +2,7 @@
 
 from __future__ import division, print_function
 from abc import ABCMeta, abstractmethod
+import math
 import sys
 import os.path
 import click
@@ -11,22 +12,6 @@ from matplotlib.backends.backend_pdf import PdfPages
 from tabulate import tabulate
 from .. import (far_threshold, plot, utils, ppndf)
 
-LINESTYLES = [
-    (0, ()),                    #solid
-    (0, (4, 4)),                #dashed
-    (0, (1, 5)),                #dotted
-    (0, (3, 5, 1, 5)),          #dashdotted
-    (0, (3, 5, 1, 5, 1, 5)),    #dashdotdotted
-    (0, (5, 1)),                #densely dashed
-    (0, (1, 1)),                #densely dotted
-    (0, (3, 1, 1, 1)),          #densely dashdotted
-    (0, (3, 1, 1, 1, 1, 1)),    #densely dashdotdotted
-    (0, (5, 10)),               #loosely dashed
-    (0, (3, 10, 1, 10)),        #loosely dashdotted
-    (0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
-    (0, (1, 10))                #loosely dotted
-]
-
 class MeasureBase(object):
     """Base class for metrics and plots.
     This abstract class define the framework to plot or compute metrics from a
@@ -77,28 +62,36 @@ class MeasureBase(object):
         systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
         (after the loop).
         """
-        #init matplotlib, log files, ...
+        # init matplotlib, log files, ...
         self.init_process()
-        #iterates through the different systems and feed `compute`
-        #with the dev (and eval) scores of each system
+        # iterates through the different systems and feed `compute`
+        # with the dev (and eval) scores of each system
         # Note that more than one dev or eval scores score can be passed to
         # each system
         for idx in range(self.n_systems):
+            # load scores for each system: get the corresponding arrays and 
+            # base-name of files
             input_scores, input_names = self._load_files(
+                # Scores are given as followed:
+                # SysA-dev SysA-eval ... SysA-XX  SysB-dev SysB-eval ... SysB-XX
+                # ------------------------------  ------------------------------
+                #   First set of `self._min_arg`     Second set of input files
+                #     input files starting at               for SysB
+                #    index idx * self._min_arg
                 self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
             )
             self.compute(idx, input_scores, input_names)
-        #setup final configuration, plotting properties, ...
+        # setup final configuration, plotting properties, ...
         self.end_process()
 
-    #protected functions that need to be overwritten
+    # protected functions that need to be overwritten
     def init_process(self):
         """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
         before iterating through the different systems.
         Should reimplemented in derived classes"""
         pass
 
-    #Main computations are done here in the subclasses
+    # Main computations are done here in the subclasses
     @abstractmethod
     def compute(self, idx, input_scores, input_names):
         """Compute metrics or plots from the given scores provided by
@@ -116,7 +109,7 @@ class MeasureBase(object):
         """
         pass
 
-    #Things to do after the main iterative computations are done
+    # Things to do after the main iterative computations are done
     @abstractmethod
     def end_process(self):
         """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
@@ -124,7 +117,7 @@ class MeasureBase(object):
         Should reimplemented in derived classes"""
         pass
 
-    #common protected functions
+    # common protected functions
 
     def _load_files(self, filepaths):
         ''' Load the input files and return the base names of the files
@@ -274,6 +267,12 @@ class PlotBase(MeasureBase):
         self._points = 100 if 'points' not in ctx.meta else ctx.meta['points']
         self._split = None if 'split' not in ctx.meta else ctx.meta['split']
         self._axlim = None if 'axlim' not in ctx.meta else ctx.meta['axlim']
+        self._min_dig = None
+        if 'min_far_value' in ctx.meta:
+            self._min_dig = int(math.log10(ctx.meta['min_far_value']))
+        elif self._axlim is not None:
+            self._min_dig = int(math.log10(self._axlim[0])
+                                if self._axlim[0] != 0 else 0)
         self._clayout = None if 'clayout' not in ctx.meta else\
         ctx.meta['clayout']
         self._far_at = None if 'lines_at' not in ctx.meta else\
@@ -290,6 +289,9 @@ class PlotBase(MeasureBase):
             mpl.style.use(ctx.meta['style'])
         self._nb_figs = 2 if self._eval and self._split else 1
         self._colors = utils.get_colors(self.n_systems)
+        self._line_linestyles = False if 'line_linestyles' not in ctx.meta else \
+                ctx.meta['line_linestyles']
+        self._linestyles = utils.get_linestyles(self.n_systems, self._line_linestyles)
         self._states = ['Development', 'Evaluation']
         self._title = None if 'title' not in ctx.meta else ctx.meta['title']
         self._x_label = None if 'x_label' not in ctx.meta else\
@@ -383,7 +385,10 @@ class Roc(PlotBase):
         self._y_label = self._y_label or "1 - False Negative Rate"
         #custom defaults
         if self._axlim is None:
-            self._axlim = [1e-4, 1.0, 1e-4, 1.0]
+            self._axlim = [1e-4, 1.0, 0, 1.0]
+
+        if self._min_dig is not None:
+            self._axlim[0] = math.pow(10, self._min_dig)
 
     def compute(self, idx, input_scores, input_names):
         ''' Plot ROC for dev and eval data using
@@ -397,20 +402,20 @@ class Roc(PlotBase):
 
         mpl.figure(1)
         if self._eval:
-            linestyle = '-' if not self._split else LINESTYLES[idx % 14]
             plot.roc_for_far(
                 dev_neg, dev_pos,
-                color=self._colors[idx], linestyle=linestyle,
+                far_values=plot.log_values(self._min_dig or -4),
+                color=self._colors[idx], linestyle=self._linestyles[idx],
                 label=self._label('development', dev_file, idx)
             )
-            linestyle = '--'
             if self._split:
                 mpl.figure(2)
-                linestyle = LINESTYLES[idx % 14]
 
+            linestyle = '--' if not self._split else self._linestyles[idx]
             plot.roc_for_far(
-                eval_neg, eval_pos,
-                color=self._colors[idx], linestyle=linestyle,
+                eval_neg, eval_pos, linestyle=linestyle,
+                far_values=plot.log_values(self._min_dig or -4),
+                color=self._colors[idx],
                 label=self._label('eval', eval_file, idx)
             )
             if self._far_at is not None:
@@ -424,7 +429,8 @@ class Roc(PlotBase):
         else:
             plot.roc_for_far(
                 dev_neg, dev_pos,
-                color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
+                far_values=plot.log_values(self._min_dig or -4),
+                color=self._colors[idx], linestyle=self._linestyles[idx],
                 label=self._label('development', dev_file, idx)
             )
 
@@ -441,6 +447,12 @@ class Det(PlotBase):
         if self._x_rotation is None:
             self._x_rotation = 50
 
+        if self._axlim is None:
+            self._axlim = [0.01, 99, 0.01, 99]
+
+        if self._min_dig is not None:
+            self._axlim[0] = math.pow(10, self._min_dig) * 100
+
     def compute(self, idx, input_scores, input_names):
         ''' Plot DET for dev and eval data using
         :py:func:`bob.measure.plot.det`'''
@@ -453,15 +465,14 @@ class Det(PlotBase):
 
         mpl.figure(1)
         if self._eval and eval_neg is not None:
-            linestyle = '-' if not self._split else LINESTYLES[idx % 14]
             plot.det(
                 dev_neg, dev_pos, self._points, color=self._colors[idx],
-                linestyle=linestyle,
+                linestyle=self._linestyles[idx],
                 label=self._label('development', dev_file, idx)
             )
             if self._split:
                 mpl.figure(2)
-            linestyle = '--' if not self._split else LINESTYLES[idx % 14]
+            linestyle = '--' if not self._split else self._linestyles[idx]
             plot.det(
                 eval_neg, eval_pos, self._points, color=self._colors[idx],
                 linestyle=linestyle,
@@ -478,15 +489,12 @@ class Det(PlotBase):
         else:
             plot.det(
                 dev_neg, dev_pos, self._points, color=self._colors[idx],
-                linestyle=LINESTYLES[idx % 14],
+                linestyle=self._linestyles[idx],
                 label=self._label('development', dev_file, idx)
             )
 
     def _set_axis(self):
-        if self._axlim is not None and None not in self._axlim:
-            plot.det_axis(self._axlim)
-        else:
-            plot.det_axis([0.01, 99, 0.01, 99])
+        plot.det_axis(self._axlim)
 
 class Epc(PlotBase):
     ''' Handles the plotting of EPC '''
@@ -513,7 +521,7 @@ class Epc(PlotBase):
 
         plot.epc(
             dev_neg, dev_pos, eval_neg, eval_pos, self._points,
-            color=self._colors[idx], linestyle=LINESTYLES[idx % 14],
+            color=self._colors[idx], linestyle=self._linestyles[idx],
             label=self._label(
                 'curve', dev_file + "_" + eval_file, idx
             )

bob/measure/test_script.py

@@ -30,7 +30,7 @@ def test_metrics():
         assert result.exit_code == 0
     with runner.isolated_filesystem():
         result = runner.invoke(
-            commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-l',
+            commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-Z',
                               'A,B']
         )
         assert result.exit_code == 0, (result.exit_code, result.output)

bob/measure/utils.py

@@ -137,6 +137,41 @@ def get_colors(n):
 
     return ['C0','C1','C2','C3','C4','C5','C6','C7','C8','C9']
 
+def get_linestyles(n, on=True):
+    """Get a list of matplotlib linestyles
+
+    Parameters
+    ----------
+    n : :obj:`int`
+        Number of linestyles to output
+
+    Returns
+    -------
+    :any:`list`
+        list of linestyles
+    """
+    if not on:
+        return [None] * n
+
+    list_linestyles = [
+        (0, ()),                    #solid
+        (0, (1, 1)),                #densely dotted
+        (0, (5, 5)),                #dashed
+        (0, (5, 1)),                #densely dashed
+        (0, (3, 1, 1, 1, 1, 1)),    #densely dashdotdotted
+        (0, (3, 10, 1, 10, 1, 10)), #loosely dashdotdotted
+        (0, (3, 5, 1, 5, 1, 5)),    #dashdotdotted
+        (0, (3, 1, 1, 1)),          #densely dashdotted
+        (0, (1, 5)),                #dotted
+        (0, (3, 5, 1, 5)),          #dashdotted
+        (0, (5, 10)),               #loosely dashed
+        (0, (3, 10, 1, 10)),        #loosely dashdotted
+        (0, (1, 10))                #loosely dotted
+    ]
+    while n > len(list_linestyles):
+        list_linestyles += list_linestyles
+    return list_linestyles
+
 def confidence_for_indicator_variable(x, n, alpha=0.05):
     '''Calculates the confidence interval for proportion estimates
     The Clopper-Pearson interval method is used for estimating the confidence