Remove bob vuln metrics and evaluate commands

since they were not well defined and we do
not know what should be in there.

rename --hlines-at to --fnmr in bob vuln roc,det commands

small nit-pick fixes overall

bob/pad/base/script/pad_commands.py

@@ -88,7 +88,7 @@ def evaluate(ctx, scores, evaluation, **kwargs):
     common_options.MULTI_METRICS_HELP.format(
         names='FtA, APCER, BPCER, FAR, FRR, ACER',
         criteria=CRITERIA, score_format=SCORE_FORMAT,
-        command='bob measure multi-metrics'),
+        command='bob pad multi-metrics'),
     criteria=CRITERIA)
 def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
   ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)

bob/pad/base/script/vuln_figure.py

 '''Runs error analysis on score sets, outputs metrics and plots'''
 
-import math
 import click
 import numpy as np
 import matplotlib.pyplot as mpl
 import bob.measure.script.figure as measure_figure
-import bob.bio.base.script.figure as bio_figure
-from tabulate import tabulate
 from bob.measure.utils import get_fta_list
 from bob.measure import (
-    frr_threshold, far_threshold, eer_threshold, min_hter_threshold, farfrr,
-    epc, ppndf, min_weighted_error_rate_threshold
+    frr_threshold, far_threshold, farfrr,
+    ppndf, min_weighted_error_rate_threshold
 )
-from bob.measure.plot import (det, det_axis, roc_for_far, log_values, epc)
+from bob.measure import plot
 from . import error_utils
 
 
-class Metrics(measure_figure.Metrics):
-    def __init__(self, ctx, scores, evaluation, func_load):
-        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
-
-    ''' Compute metrics from score files'''
-
-    def compute(self, idx, input_scores, input_names):
-        ''' Compute metrics for the given criteria'''
-        # extract pos and negative and remove NaNs
-        neg_list, pos_list, _ = get_fta_list(input_scores)
-        dev_neg, dev_pos = neg_list[0], pos_list[0]
-        criter = self._criterion or 'eer'
-        threshold = error_utils.calc_threshold(criter, dev_neg, dev_pos) \
-            if self._thres is None else self._thres[idx]
-        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
-        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
-        title = self._legends[idx] if self._legends is not None else None
-        headers = ['' or title, '%s (threshold=%.2g)' %
-                   (criter.upper(), threshold)]
-        rows = []
-        rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
-        rows.append(['FNMR (%)', '{:>5.1f}%'.format(frr * 100)])
-        rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
-        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
-        click.echo(
-            tabulate(rows, headers, self._tablefmt),
-            file=self.log_file
-        )
-
-
 def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
     # plot a dot on threshold versus IAPMR line and show IAPMR as a number
     axlim = mpl.axis()
@@ -185,7 +152,7 @@ class Epc(PadPlot):
         mpl.gcf().clear()
         mpl.grid()
 
-        epc(
+        plot.epc(
             licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos,
             self._points,
             color='C0', linestyle=self._linestyles[idx],
@@ -228,7 +195,6 @@ class Epc(PadPlot):
             ax1.tick_params(axis='y', colors='C0')
             ax1.spines['left'].set_color('C0')
 
-
         title = self._legends[idx] if self._legends is not None else self._title
         if title.replace(' ', ''):
             mpl.title(title)
@@ -448,7 +414,7 @@ class BaseVulnDetRoc(PadPlot):
         super(BaseVulnDetRoc, self).__init__(
             ctx, scores, evaluation, func_load)
         self._no_spoof = no_spoof
-        self._hlines_at = ctx.meta.get('hlines_at', [])
+        self._fnmrs_at = ctx.meta.get('fnmr', [])
         self._real_data = True if real_data is None else real_data
         self._legend_loc = None
 
@@ -469,7 +435,7 @@ class BaseVulnDetRoc(PadPlot):
         if not self._no_spoof and spoof_neg is not None:
             ax1 = mpl.gca()
             ax2 = ax1.twiny()
-            ax2.set_xlabel('IAPMR', color='C3')
+            ax2.set_xlabel('IAPMR (%)', color='C3')
             ax2.set_xticklabels(ax2.get_xticks())
             ax2.tick_params(axis='x', colors='C3')
             ax2.xaxis.label.set_color('C3')
@@ -487,10 +453,10 @@ class BaseVulnDetRoc(PadPlot):
             )
             mpl.sca(ax1)
 
-        if self._hlines_at is None:
+        if self._fnmrs_at is None:
             return
 
-        for line in self._hlines_at:
+        for line in self._fnmrs_at:
             thres_baseline = frr_threshold(licit_neg, licit_pos, line)
 
             axlim = mpl.axis()
@@ -547,7 +513,6 @@ class BaseVulnDetRoc(PadPlot):
                 label=label_spoof
             )  # FAR point, spoof scenario
 
-
     def end_process(self):
         ''' Set title, legend, axis labels, grid colors, save figures and
         close pdf is needed '''
@@ -594,9 +559,9 @@ class DetVuln(BaseVulnDetRoc):
     def __init__(self, ctx, scores, evaluation, func_load, real_data,
                  no_spoof):
         super(DetVuln, self).__init__(ctx, scores, evaluation, func_load,
-                                  real_data, no_spoof)
-        self._x_label = self._x_label or "FMR"
-        self._y_label = self._y_label or "FNMR"
+                                      real_data, no_spoof)
+        self._x_label = self._x_label or "FMR (%)"
+        self._y_label = self._y_label or "FNMR (%)"
         add = ''
         if not self._no_spoof:
             add = " and overlaid SPOOF scenario"
@@ -605,16 +570,16 @@ class DetVuln(BaseVulnDetRoc):
 
     def _set_axis(self):
         if self._axlim is not None and None not in self._axlim:
-            det_axis(self._axlim)
+            plot.det_axis(self._axlim)
         else:
-            det_axis([0.01, 99, 0.01, 99])
+            plot.det_axis([0.01, 99, 0.01, 99])
 
     def _get_farfrr(self, x, y, thres):
         points = farfrr(x, y, thres)
         return points, [ppndf(i) for i in points]
 
     def _plot(self, x, y, points, **kwargs):
-        det(
+        plot.det(
             x, y, points,
             color=kwargs.get('color'),
             linestyle=kwargs.get('linestyle'),
@@ -639,9 +604,9 @@ class RocVuln(BaseVulnDetRoc):
         self._legend_loc = self._legend_loc or best_legend
 
     def _plot(self, x, y, points, **kwargs):
-        roc_for_far(
+        plot.roc_for_far(
             x, y,
-            far_values=log_values(self._min_dig or -4),
+            far_values=plot.log_values(self._min_dig or -4),
             CAR=self._semilogx,
             color=kwargs.get('color'), linestyle=kwargs.get('linestyle'),
             label=kwargs.get('label')
@@ -694,8 +659,8 @@ class FmrIapmr(PadPlot):
         title = self._title if self._title is not None else "FMR vs IAPMR"
         if title.replace(' ', ''):
             mpl.title(title)
-        mpl.xlabel(self._x_label or "FMR (%)")
-        mpl.ylabel(self._y_label or "IAPMR (%)")
+        mpl.xlabel(self._x_label or "FMR")
+        mpl.ylabel(self._y_label or "IAPMR")
         mpl.grid(True, color=self._grid_color)
         if self._disp_legend:
             mpl.legend(loc=self._legend_loc)

bob/pad/base/test/test_commands.py

-import sys
-import click
 from click.testing import CliRunner
 import pkg_resources
 from ..script import (pad_commands, vuln_commands)
 
 
-
 def test_det_pad():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -19,7 +16,6 @@ def test_det_pad():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
 def test_det_vuln():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -31,7 +27,7 @@ def test_det_vuln():
                                                  'data/spoof/scores-eval')
     runner = CliRunner()
     with runner.isolated_filesystem():
-        result = runner.invoke(vuln_commands.det, ['-hla', '0.2',
+        result = runner.invoke(vuln_commands.det, ['-fnmr', '0.2',
                                                    '-o',
                                                    'DET.pdf',
                                                    licit_dev, licit_test,
@@ -39,7 +35,6 @@ def test_det_vuln():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
 def test_fmr_iapmr_vuln():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -64,7 +59,6 @@ def test_fmr_iapmr_vuln():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
 def test_hist_pad():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -96,7 +90,6 @@ def test_hist_pad():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
 def test_hist_vuln():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -123,28 +116,6 @@ def test_hist_vuln():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
-
-def test_metrics_vuln():
-    licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
-                                                'data/licit/scores-dev')
-    licit_test = pkg_resources.resource_filename('bob.pad.base.test',
-                                                 'data/licit/scores-eval')
-    spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
-                                                'data/spoof/scores-dev')
-    spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
-                                                 'data/spoof/scores-eval')
-    runner = CliRunner()
-    with runner.isolated_filesystem():
-        result = runner.invoke(
-            vuln_commands.metrics,
-            ['--criterion', 'eer', licit_dev, licit_test,
-             spoof_dev, spoof_test]
-        )
-        assert result.exit_code == 0, (result.exit_code, result.output)
-
-
-
 def test_metrics_pad():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -159,7 +130,6 @@ def test_metrics_pad():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
 def test_epc_vuln():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
@@ -199,7 +169,7 @@ def test_epsc_vuln():
                                ['--output', 'epsc.pdf',
                                 licit_dev, licit_test,
                                 spoof_dev, spoof_test])
-        assert result.exit_code == 0, (result.exit_code, result.output)
+        assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
 
         result = runner.invoke(vuln_commands.epsc,
                                ['--output', 'epsc.pdf', '-I',
@@ -221,8 +191,7 @@ def test_epsc_vuln():
         assert result.exit_code == 0, (result.exit_code, result.output)
 
 
-
-def test_evaluate_vuln():
+def test_evaluate_pad():
     licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
                                                 'data/licit/scores-dev')
     licit_test = pkg_resources.resource_filename('bob.pad.base.test',
@@ -236,20 +205,3 @@ def test_evaluate_vuln():
         result = runner.invoke(pad_commands.evaluate,
                                [licit_dev, licit_test, spoof_dev, spoof_test])
         assert result.exit_code == 0, (result.exit_code, result.output)
-
-
-
-def test_evaluate_vuln():
-    licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
-                                                'data/licit/scores-dev')
-    licit_test = pkg_resources.resource_filename('bob.pad.base.test',
-                                                 'data/licit/scores-eval')
-    spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
-                                                'data/spoof/scores-dev')
-    spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
-                                                 'data/spoof/scores-eval')
-    runner = CliRunner()
-    with runner.isolated_filesystem():
-        result = runner.invoke(vuln_commands.evaluate,
-                               [licit_dev, licit_test, spoof_dev, spoof_test])
-        assert result.exit_code == 0, (result.exit_code, result.output)

setup.py

@@ -151,7 +151,6 @@ setup(
 
         # bob vuln scripts
         'bob.vuln.cli': [
-            'metrics          = bob.pad.base.script.vuln_commands:metrics',
             'hist             = bob.pad.base.script.vuln_commands:hist',
             'det              = bob.pad.base.script.vuln_commands:det',
             'roc              = bob.pad.base.script.vuln_commands:roc',
@@ -159,7 +158,6 @@ setup(
             'epsc             = bob.pad.base.script.vuln_commands:epsc',
             'gen              = bob.pad.base.script.vuln_commands:gen',
             'fmr_iapmr        = bob.pad.base.script.vuln_commands:fmr_iapmr',
-            'evaluate         = bob.pad.base.script.vuln_commands:evaluate',
         ],
 
     },