Add commands to compute bio specific metrics

bob/bio/base/script/commands.py

@@ -8,22 +8,22 @@ from ..score import load
 from bob.measure.script import common_options
 from bob.extension.scripts.click_helper import verbosity_option
 
-
 FUNC_SPLIT = lambda x: load.load_files(x, load.split)
 FUNC_CMC = lambda x: load.load_files(x, load.cmc)
 
-
 @click.command()
 @common_options.scores_argument(nargs=-1)
 @common_options.table_option()
 @common_options.test_option()
 @common_options.open_file_mode_option()
 @common_options.output_plot_metric_option()
-@common_options.criterion_option()
-@common_options.threshold_option()
+@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
+@common_options.cost_option()
+@common_options.thresholds_option()
+@common_options.far_option()
 @verbosity_option()
 @click.pass_context
-def metrics(ctx, scores, test, **kargs):
+def metrics(ctx, scores, criter, test, **kargs):
     """Prints a single output line that contains all info for a given
     criterion (eer or hter).
 
@@ -44,7 +44,10 @@ def metrics(ctx, scores, test, **kargs):
 
         $ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
     """
-    process = measure_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
+    if criter == 'rr':
+        process = bio_figure.Metrics(ctx, scores, test, FUNC_CMC)
+    else:
+        process = bio_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
     process.run()
 
 @click.command()
@@ -154,7 +157,7 @@ def epc(ctx, scores, **kargs):
 @common_options.n_bins_option()
 @common_options.criterion_option()
 @common_options.axis_fontsize_option()
-@common_options.threshold_option()
+@common_options.thresholds_option()
 @verbosity_option()
 @click.pass_context
 def hist(ctx, scores, test, **kwargs):

bob/bio/base/script/figure.py

 '''Plots and measures for bob.bio.base'''
 
+import click
 import matplotlib.pyplot as mpl
 import  bob.measure.script.figure as measure_figure
+import bob.measure
 from bob.measure import plot
+from tabulate import tabulate
 
 class Cmc(measure_figure.PlotBase):
     ''' Handles the plotting of Cmc
@@ -108,3 +111,96 @@ class Dic(measure_figure.PlotBase):
                 color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
                 label=self._label('development', dev_file, idx)
             )
+
+class Metrics(measure_figure.Metrics):
+    ''' Compute metrics from score files'''
+    def init_process(self):
+        if self._criter == 'rr':
+            self._thres = [None] * len(self.dev_names) if self._thres is None else \
+                    self._thres
+
+    def compute(self, idx, dev_score, dev_file=None,
+                test_score=None, test_file=None):
+        ''' Compute metrics for the given criteria'''
+        headers = ['', 'Development %s' % dev_file]
+        if self._test and test_score is not None:
+            headers.append('Test % s' % test_file)
+        if self._criter == 'rr':
+            rr = bob.measure.recognition_rate(dev_score, self._thres[idx])
+            dev_rr = "%.3f%%" % (100 * rr)
+            raws = [['RR', dev_rr]]
+            if self._test and test_score is not None:
+                rr = bob.measure.recognition_rate(test_score, self._thres[idx])
+                test_rr = "%.3f%%" % (100 * rr)
+                raws[0].append(test_rr)
+            click.echo(
+                tabulate(raws, headers, self._tablefmt), file=self.log_file
+            )
+        elif self._criter == 'mindcf':
+            if 'cost' in self._ctx.meta:
+                cost = 0.99 if 'cost' not in self._ctx.meta else\
+                        self._ctx.meta['cost']
+            threshold = bob.measure.min_weighted_error_rate_threshold(
+                dev_score[0], dev_score[1], cost
+            ) if self._thres is None else self._thres[idx]
+            if self._thres is None:
+                click.echo(
+                    "[minDCF - Cost:%f] Threshold on Development set `%s`: %e"\
+                    % (cost, dev_file, threshold),
+                    file=self.log_file
+                )
+            else:
+                click.echo(
+                    "[minDCF] User defined Threshold: %e" %  threshold,
+                    file=self.log_file
+                )
+            # apply threshold to development set
+            far, frr = bob.measure.farfrr(
+                dev_score[0], dev_score[1], threshold
+            )
+            dev_far_str = "%.3f%%" % (100 * far)
+            dev_frr_str = "%.3f%%" % (100 * frr)
+            dev_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
+            raws = [['FAR', dev_far_str],
+                    ['FRR', dev_frr_str],
+                    ['minDCF', dev_mindcf_str]]
+            if self._test and test_score is not None:
+                # apply threshold to development set
+                far, frr = bob.measure.farfrr(
+                    test_score[0], test_score[1], threshold
+                )
+                test_far_str = "%.3f%%" % (100 * far)
+                test_frr_str = "%.3f%%" % (100 * frr)
+                test_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
+                raws[0].append(test_far_str)
+                raws[1].append(test_frr_str)
+                raws[2].append(test_mindcf_str)
+            click.echo(
+                tabulate(raws, headers, self._tablefmt), file=self.log_file
+            )
+        elif self._criter == 'cllr':
+            cllr = bob.measure.calibration.cllr(dev_score[0], dev_score[1])
+            min_cllr = bob.measure.calibration.min_cllr(
+                dev_score[0], dev_score[1]
+            )
+            dev_cllr_str = "%.3f%%" % cllr
+            dev_min_cllr_str = "%.3f%%" % min_cllr
+            raws = [['Cllr', dev_cllr_str],
+                    ['minCllr', dev_min_cllr_str]]
+            if self._test and test_score is not None:
+                cllr = bob.measure.calibration.cllr(test_score[0],
+                                                    test_score[1])
+                min_cllr = bob.measure.calibration.min_cllr(
+                    test_score[0], test_score[1]
+                )
+                test_cllr_str = "%.3f%%" % cllr
+                test_min_cllr_str = "%.3f%%" % min_cllr
+                raws[0].append(test_cllr_str)
+                raws[1].append(test_min_cllr_str)
+                click.echo(
+                    tabulate(raws, headers, self._tablefmt), file=self.log_file
+                )
+        else:
+            super(Metrics, self).compute(
+                idx, dev_score, dev_file, test_score, test_file
+            )

bob/bio/base/test/test_commands.py

@@ -37,10 +37,53 @@ def test_metrics():
         assert result.exit_code == 0
     with runner.isolated_filesystem():
         result = runner.invoke(
-            commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
+            commands.metrics, ['-l', 'tmp', '--test', dev1, test2]
         )
         assert result.exit_code == 0
 
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp', '-t', '-T', '0.1',
+                               '--criter', 'mindcf', '--cost', 0.9,
+                               dev1, test2]
+        )
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp',
+                               '--criter', 'mindcf', '--cost', 0.9,
+                               dev1]
+        )
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-t', '--criter', 'cllr', dev1, test2]
+        )
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp', '--criter', 'cllr', '--cost', 0.9,
+                               dev1]
+        )
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-t', '--criter', 'rr', '-T',
+                               '0.1', dev1, test2]
+        )
+        assert result.exit_code == 0
+
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            commands.metrics, ['-l', 'tmp', '--criter', 'rr', dev1, dev2]
+        )
+        assert result.exit_code == 0
+
+
 def test_roc():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/dev-4col.txt')