From 73a7b02e0672021da13bf174588aa0821cda9eb4 Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Fri, 6 Apr 2018 16:42:12 +0200
Subject: [PATCH] Add commands to compute bio specific metrics
---
bob/bio/base/script/commands.py | 17 +++---
bob/bio/base/script/figure.py | 96 ++++++++++++++++++++++++++++++
bob/bio/base/test/test_commands.py | 45 +++++++++++++-
3 files changed, 150 insertions(+), 8 deletions(-)
diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
index 7fe761e8..6a205222 100644
--- a/bob/bio/base/script/commands.py
+++ b/bob/bio/base/script/commands.py
@@ -8,22 +8,22 @@ from ..score import load
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
-
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
FUNC_CMC = lambda x: load.load_files(x, load.cmc)
-
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
-@common_options.criterion_option()
-@common_options.threshold_option()
+@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
+@common_options.cost_option()
+@common_options.thresholds_option()
+@common_options.far_option()
@verbosity_option()
@click.pass_context
-def metrics(ctx, scores, test, **kargs):
+def metrics(ctx, scores, criter, test, **kargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
@@ -44,7 +44,10 @@ def metrics(ctx, scores, test, **kargs):
$ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
"""
- process = measure_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
+ if criter == 'rr':
+ process = bio_figure.Metrics(ctx, scores, test, FUNC_CMC)
+ else:
+ process = bio_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
process.run()
@click.command()
@@ -154,7 +157,7 @@ def epc(ctx, scores, **kargs):
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
-@common_options.threshold_option()
+@common_options.thresholds_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kwargs):
diff --git a/bob/bio/base/script/figure.py b/bob/bio/base/script/figure.py
index 2ab16e3f..30d5fdef 100644
--- a/bob/bio/base/script/figure.py
+++ b/bob/bio/base/script/figure.py
@@ -1,8 +1,11 @@
'''Plots and measures for bob.bio.base'''
+import click
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
+import bob.measure
from bob.measure import plot
+from tabulate import tabulate
class Cmc(measure_figure.PlotBase):
''' Handles the plotting of Cmc
@@ -108,3 +111,96 @@ class Dic(measure_figure.PlotBase):
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', dev_file, idx)
)
+
+class Metrics(measure_figure.Metrics):
+ ''' Compute metrics from score files'''
+ def init_process(self):
+ if self._criter == 'rr':
+ self._thres = [None] * len(self.dev_names) if self._thres is None else \
+ self._thres
+
+ def compute(self, idx, dev_score, dev_file=None,
+ test_score=None, test_file=None):
+ ''' Compute metrics for the given criteria'''
+ headers = ['', 'Development %s' % dev_file]
+ if self._test and test_score is not None:
+ headers.append('Test % s' % test_file)
+ if self._criter == 'rr':
+ rr = bob.measure.recognition_rate(dev_score, self._thres[idx])
+ dev_rr = "%.3f%%" % (100 * rr)
+ raws = [['RR', dev_rr]]
+ if self._test and test_score is not None:
+ rr = bob.measure.recognition_rate(test_score, self._thres[idx])
+ test_rr = "%.3f%%" % (100 * rr)
+ raws[0].append(test_rr)
+ click.echo(
+ tabulate(raws, headers, self._tablefmt), file=self.log_file
+ )
+ elif self._criter == 'mindcf':
+ if 'cost' in self._ctx.meta:
+ cost = 0.99 if 'cost' not in self._ctx.meta else\
+ self._ctx.meta['cost']
+ threshold = bob.measure.min_weighted_error_rate_threshold(
+ dev_score[0], dev_score[1], cost
+ ) if self._thres is None else self._thres[idx]
+ if self._thres is None:
+ click.echo(
+ "[minDCF - Cost:%f] Threshold on Development set `%s`: %e"\
+ % (cost, dev_file, threshold),
+ file=self.log_file
+ )
+ else:
+ click.echo(
+ "[minDCF] User defined Threshold: %e" % threshold,
+ file=self.log_file
+ )
+ # apply threshold to development set
+ far, frr = bob.measure.farfrr(
+ dev_score[0], dev_score[1], threshold
+ )
+ dev_far_str = "%.3f%%" % (100 * far)
+ dev_frr_str = "%.3f%%" % (100 * frr)
+ dev_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
+ raws = [['FAR', dev_far_str],
+ ['FRR', dev_frr_str],
+ ['minDCF', dev_mindcf_str]]
+ if self._test and test_score is not None:
+ # apply threshold to development set
+ far, frr = bob.measure.farfrr(
+ test_score[0], test_score[1], threshold
+ )
+ test_far_str = "%.3f%%" % (100 * far)
+ test_frr_str = "%.3f%%" % (100 * frr)
+ test_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
+ raws[0].append(test_far_str)
+ raws[1].append(test_frr_str)
+ raws[2].append(test_mindcf_str)
+ click.echo(
+ tabulate(raws, headers, self._tablefmt), file=self.log_file
+ )
+ elif self._criter == 'cllr':
+ cllr = bob.measure.calibration.cllr(dev_score[0], dev_score[1])
+ min_cllr = bob.measure.calibration.min_cllr(
+ dev_score[0], dev_score[1]
+ )
+ dev_cllr_str = "%.3f%%" % cllr
+ dev_min_cllr_str = "%.3f%%" % min_cllr
+ raws = [['Cllr', dev_cllr_str],
+ ['minCllr', dev_min_cllr_str]]
+ if self._test and test_score is not None:
+ cllr = bob.measure.calibration.cllr(test_score[0],
+ test_score[1])
+ min_cllr = bob.measure.calibration.min_cllr(
+ test_score[0], test_score[1]
+ )
+ test_cllr_str = "%.3f%%" % cllr
+ test_min_cllr_str = "%.3f%%" % min_cllr
+ raws[0].append(test_cllr_str)
+ raws[1].append(test_min_cllr_str)
+ click.echo(
+ tabulate(raws, headers, self._tablefmt), file=self.log_file
+ )
+ else:
+ super(Metrics, self).compute(
+ idx, dev_score, dev_file, test_score, test_file
+ )
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
index 159927f4..70d4c32f 100644
--- a/bob/bio/base/test/test_commands.py
+++ b/bob/bio/base/test/test_commands.py
@@ -37,10 +37,53 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
+ commands.metrics, ['-l', 'tmp', '--test', dev1, test2]
)
assert result.exit_code == 0
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp', '-t', '-T', '0.1',
+ '--criter', 'mindcf', '--cost', 0.9,
+ dev1, test2]
+ )
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp',
+ '--criter', 'mindcf', '--cost', 0.9,
+ dev1]
+ )
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-t', '--criter', 'cllr', dev1, test2]
+ )
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp', '--criter', 'cllr', '--cost', 0.9,
+ dev1]
+ )
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-t', '--criter', 'rr', '-T',
+ '0.1', dev1, test2]
+ )
+ assert result.exit_code == 0
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ commands.metrics, ['-l', 'tmp', '--criter', 'rr', dev1, dev2]
+ )
+ assert result.exit_code == 0
+
+
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
--
GitLab