[script][figure,vuln_commands] Only use licit and spoof for det and roc.

Horizontal line for FNMR

bob/pad/base/script/vuln_commands.py

@@ -12,7 +12,7 @@ from bob.measure.script import common_options
 from bob.extension.scripts.click_helper import (verbosity_option,
                                                 open_file_mode_option,
                                                bool_option,
-                                               AliasedGroup)
+                                               AliasedGroup, list_float_option)
 from bob.core import random
 from bob.io.base import create_directories_safe
 from bob.bio.base.score import load
@@ -23,6 +23,15 @@ NUM_ZEIMPOSTORS = 5000
 NUM_PA = 5000
 
 
+def hlines_at_option(dflt=' ', **kwargs):
+    '''Get option to draw const FNMRlines'''
+    return list_float_option(
+        name='hlines-at', short_name='hla',
+        desc='If given, draw horizontal lines at the given axis positions. '
+        'Your values must be separated with a comma (,) without space. '
+        'This option works in ROC and DET curves.',
+        nitems=None, dflt=dflt, **kwargs
+    )
 
 
 def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
@@ -101,7 +110,7 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
 
 
 @click.command()
-@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
+@common_options.scores_argument(min_arg=2, nargs=-1)
 @common_options.output_plot_file_option(default_out='vuln_roc.pdf')
 @common_options.legends_option()
 @common_options.no_legend_option()
@@ -116,37 +125,32 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
 @common_options.x_rotation_option(dflt=45)
 @common_options.x_label_option()
 @common_options.y_label_option()
-@click.option('-c', '--criteria', default=None, show_default=True,
-              help='Criteria for threshold selection',
-              type=click.Choice(('eer', 'min-hter')))
 @click.option('--real-data/--no-real-data', default=True, show_default=True,
               help='If False, will annotate the plots hypothetically, instead '
               'of with real data values of the calculated error rates.')
+@hlines_at_option()
 @click.pass_context
-def roc(ctx, scores, criteria, real_data, **kwargs):
+def roc(ctx, scores, real_data, **kwargs):
   """Plot ROC
 
-  You need to provide 4 scores
+  You need to provide 2 scores
   files for each vulnerability system in this order:
 
   \b
-  * licit development scores
-  * licit evaluation scores
-  * spoof development scores
-  * spoof evaluation scores
+  * licit scores
+  * spoof scores
 
   Examples:
-      $ bob vuln roc -v dev-scores eval-scores
+      $ bob vuln roc -v licit-scores spoof-scores
 
-      $ bob vuln roc -v {licit,spoof}/scores-{dev,eval}
+      $ bob vuln roc -v scores-{licit,spoof}
   """
-  process = figure.RocVuln(ctx, scores, True, load.split, criteria, real_data,
-                       False)
+  process = figure.RocVuln(ctx, scores, True, load.split, real_data, False)
   process.run()
 
 
 @click.command()
-@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
+@common_options.scores_argument(min_arg=2, nargs=-1)
 @common_options.output_plot_file_option(default_out='vuln_det.pdf')
 @common_options.legends_option()
 @common_options.no_legend_option()
@@ -160,32 +164,28 @@ def roc(ctx, scores, criteria, real_data, **kwargs):
 @common_options.x_rotation_option(dflt=45)
 @common_options.x_label_option()
 @common_options.y_label_option()
-@click.option('-c', '--criteria', default=None, show_default=True,
-              help='Criteria for threshold selection',
-              type=click.Choice(('eer', 'min-hter')))
 @click.option('--real-data/--no-real-data', default=True, show_default=True,
               help='If False, will annotate the plots hypothetically, instead '
               'of with real data values of the calculated error rates.')
+@hlines_at_option()
 @click.pass_context
-def det(ctx, scores, criteria, real_data, **kwargs):
+def det(ctx, scores, real_data, **kwargs):
   """Plot DET
 
-  You need to provide 4 scores
-  files for each vuln system in this order:
+
+  You need to provide 2 scores
+  files for each vulnerability system in this order:
 
   \b
-  * licit development scores
-  * licit evaluation scores
-  * spoof development scores
-  * spoof evaluation scores
+  * licit scores
+  * spoof scores
 
   Examples:
-      $ bob vuln det -v dev-scores eval-scores
+      $ bob vuln det -v licit-scores spoof-scores
 
-      $ bob vuln det -v {licit,spoof}/scores-{dev,eval}
+      $ bob vuln det -v scores-{licit,spoof}
   """
-  process = figure.Det(ctx, scores, True, load.split, criteria, real_data,
-                       False)
+  process = figure.Det(ctx, scores, True, load.split, real_data, False)
   process.run()
 
 

bob/pad/base/script/vuln_figure.py

 '''Runs error analysis on score sets, outputs metrics and plots'''
 
+import math
 import click
 import numpy as np
 import matplotlib.pyplot as mpl
@@ -434,32 +435,30 @@ class Epsc3D(Epsc):
 class BaseVulnDetRoc(PadPlot):
     '''Base for DET and ROC'''
 
-    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
+    def __init__(self, ctx, scores, evaluation, func_load, real_data,
                  no_spoof):
         super(BaseVulnDetRoc, self).__init__(
             ctx, scores, evaluation, func_load)
         self._no_spoof = no_spoof
-        self._criteria = criteria or 'eer'
+        self._hlines_at = ctx.meta.get('hlines_at', [])
         self._real_data = True if real_data is None else real_data
         self._legend_loc = None
 
     def compute(self, idx, input_scores, input_names):
         ''' Implements plots'''
-        licit_dev_neg = input_scores[0][0]
-        licit_dev_pos = input_scores[0][1]
-        licit_eval_neg = input_scores[1][0]
-        licit_eval_pos = input_scores[1][1]
-        spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
-        spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
+        licit_neg = input_scores[0][0]
+        licit_pos = input_scores[0][1]
+        spoof_neg = input_scores[1][0]
+        spoof_pos = input_scores[1][1]
         self._plot(
-            licit_eval_neg,
-            licit_eval_pos,
+            licit_neg,
+            licit_pos,
             self._points,
             color='C0',
             linestyle='-',
             label=self._label("licit", input_names[0], idx)
         )
-        if not self._no_spoof and spoof_eval_neg is not None:
+        if not self._no_spoof and spoof_neg is not None:
             ax1 = mpl.gca()
             ax2 = ax1.twiny()
             ax2.set_xlabel('IAPMR', color='C3')
@@ -467,114 +466,117 @@ class BaseVulnDetRoc(PadPlot):
             ax2.tick_params(axis='x', colors='C3')
             ax2.xaxis.label.set_color('C3')
             ax2.spines['top'].set_color('C3')
+            ax2.spines['bottom'].set_color('C0')
+            ax1.xaxis.label.set_color('C0')
+            ax1.tick_params(axis='x', colors='C0')
             self._plot(
-                spoof_eval_neg,
-                spoof_eval_pos,
+                spoof_neg,
+                spoof_pos,
                 self._points,
                 color='C3',
                 linestyle=':',
-                label=self._label("spoof", input_names[3], idx)
+                label=self._label("spoof", input_names[1], idx)
             )
             mpl.sca(ax1)
 
-        if self._criteria is None or self._no_spoof:
+        if self._hlines_at is None:
             return
 
-        thres_baseline = calc_threshold(
-            self._criteria, licit_dev_neg, licit_dev_pos
-        )
+        for line in self._hlines_at:
+            print (line)
+            thres_baseline = frr_threshold(licit_neg, licit_pos, line)
 
-        axlim = mpl.axis()
+            axlim = mpl.axis()
 
-        farfrr_licit, farfrr_licit_det = self._get_farfrr(
-            licit_eval_neg, licit_eval_pos,
-            thres_baseline
-        )
-        if farfrr_licit is None:
-            return
-        farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
-            spoof_eval_neg, spoof_eval_pos,
-            frr_threshold(spoof_eval_neg, spoof_eval_pos,
-                          farfrr_licit[1])
-        )
+            farfrr_licit, farfrr_licit_det = self._get_farfrr(
+                licit_neg, licit_pos,
+                thres_baseline
+            )
+            if farfrr_licit is None:
+                return
 
-        if not self._real_data:
-            mpl.axhline(
-                y=farfrr_licit_det[1],
-                xmin=axlim[2],
-                xmax=axlim[3],
-                color='k',
-                linestyle='--',
-                label="%s @ EER" % self._y_label)
-        else:
-            mpl.axhline(
-                y=farfrr_licit_det[1],
-                xmin=axlim[0],
-                xmax=axlim[1],
-                color='k',
-                linestyle='--',
-                label="%s = %.2f%%" %
-                (self._y_label, farfrr_licit[1] * 100))
+            farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
+                spoof_neg, spoof_pos,
+                frr_threshold(spoof_neg, spoof_pos, farfrr_licit[1])
+            )
 
-        mpl.plot(
-            farfrr_licit_det[0],
-            farfrr_licit_det[1],
-            'o',
-            color='C0',
-        )  # FAR point, licit scenario
-        mpl.plot(
-            farfrr_spoof_det[0],
-            farfrr_spoof_det[1],
-            'o',
-            color='C3',
-        )  # FAR point, spoof scenario
-
-        # annotate the FAR points
-        if farfrr_licit_det[0] < farfrr_spoof_det[0]:
-            xyannotate_licit = [
-                farfrr_licit_det[0] - 0.7,
-                farfrr_licit_det[1] - 0.4,
-            ]
-            xyannotate_spoof = [
-                0.1 + farfrr_spoof_det[0],
-                farfrr_spoof_det[1] + 0.3,
-            ]
-        else:
-            xyannotate_spoof = [
-                farfrr_licit_det[0] - 0.7,
-                farfrr_licit_det[1] - 0.4,
-            ]
-            xyannotate_licit = [
-                0.1 + farfrr_spoof_det[0],
-                farfrr_spoof_det[1] + 0.3,
-            ]
-
-        if not self._real_data:
-            mpl.annotate(
-                '%s @ operating point' % self._y_label,
-                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
-                xycoords='data',
-                xytext=(xyannotate_licit[0], xyannotate_licit[1]))
-            mpl.annotate(
-                'IAPMR @ operating point',
-                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
-                xycoords='data',
-                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
-        else:
-            mpl.annotate(
-                'FMR=%.2f%%' % (farfrr_licit[0] * 100),
-                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
-                xycoords='data',
-                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
+            if not self._real_data:
+                mpl.axhline(
+                    y=farfrr_licit_det[1],
+                    xmin=axlim[2],
+                    xmax=axlim[3],
+                    color='k',
+                    linestyle='--',
+                    label="%s @ EER" % self._y_label)
+            else:
+                mpl.axhline(
+                    y=farfrr_licit_det[1],
+                    xmin=axlim[0],
+                    xmax=axlim[1],
+                    color='k',
+                    linestyle='--',
+                    label="%s = %.2f%%" %
+                    (self._y_label, farfrr_licit[1] * 100))
+
+            mpl.plot(
+                farfrr_licit_det[0],
+                farfrr_licit_det[1],
+                'o',
                 color='C0',
-                size='small')
-            mpl.annotate(
-                'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
-                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
-                xycoords='data',
-                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
+            )  # FAR point, licit scenario
+            mpl.plot(
+                farfrr_spoof_det[0],
+                farfrr_spoof_det[1],
+                'o',
                 color='C3',
-                size='small')
+            )  # FAR point, spoof scenario
+
+            # annotate the FAR points
+            if farfrr_licit_det[0] < farfrr_spoof_det[0]:
+                xyannotate_licit = [
+                    farfrr_licit_det[0] - 0.7,
+                    farfrr_licit_det[1] - 0.4,
+                ]
+                xyannotate_spoof = [
+                    0.1 + farfrr_spoof_det[0],
+                    farfrr_spoof_det[1] + 0.3,
+                ]
+            else:
+                xyannotate_spoof = [
+                    farfrr_licit_det[0] - 0.7,
+                    farfrr_licit_det[1] - 0.4,
+                ]
+                xyannotate_licit = [
+                    0.1 + farfrr_spoof_det[0],
+                    farfrr_spoof_det[1] + 0.3,
+                ]
+
+            if not self._real_data:
+                mpl.annotate(
+                    '%s @ operating point' % self._y_label,
+                    xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
+                    xycoords='data',
+                    xytext=(xyannotate_licit[0], xyannotate_licit[1]))
+                mpl.annotate(
+                    'IAPMR @ operating point',
+                    xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
+                    xycoords='data',
+                    xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
+            else:
+                mpl.annotate(
+                    'FMR=%.2f%%' % (farfrr_licit[0] * 100),
+                    xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
+                    xycoords='data',
+                    xytext=(xyannotate_licit[0], xyannotate_licit[1]),
+                    color='C0',
+                    size='small')
+                mpl.annotate(
+                    'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
+                    xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
+                    xycoords='data',
+                    xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
+                    color='C3',
+                    size='small')
 
     def end_process(self):
         ''' Set title, legend, axis labels, grid colors, save figures and
@@ -619,10 +621,10 @@ class BaseVulnDetRoc(PadPlot):
 class DetVuln(BaseVulnDetRoc):
     '''DET for vuln'''
 
-    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
+    def __init__(self, ctx, scores, evaluation, func_load, real_data,
                  no_spoof):
         super(Det, self).__init__(ctx, scores, evaluation, func_load,
-                                  criteria, real_data, no_spoof)
+                                  real_data, no_spoof)
         self._x_label = self._x_label or "FMR"
         self._y_label = self._y_label or "FNMR"
         add = ''
@@ -653,10 +655,9 @@ class DetVuln(BaseVulnDetRoc):
 class RocVuln(BaseVulnDetRoc):
     '''ROC for vuln'''
 
-    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
-                 no_spoof):
+    def __init__(self, ctx, scores, evaluation, func_load, real_data, no_spoof):
         super(RocVuln, self).__init__(ctx, scores, evaluation, func_load,
-                                      criteria, real_data, no_spoof)
+                                      real_data, no_spoof)
         self._x_label = self._x_label or "FMR"
         self._y_label = self._y_label or "1 - FNMR"
         self._semilogx = ctx.meta.get('semilogx', True)
@@ -677,6 +678,7 @@ class RocVuln(BaseVulnDetRoc):
         )
 
 
+
 class FmrIapmr(PadPlot):
     '''FMR vs IAPMR'''