From bc9a2266e02e3baf4d0240bfcfd198248188956d Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Fri, 4 May 2018 14:25:25 +0200
Subject: [PATCH] Add vuln_metrics and change command names
---
bob/pad/base/script/figure.py | 28 ++++++++++++++++++++++++++++
bob/pad/base/script/histograms.py | 6 +++---
bob/pad/base/script/metrics.py | 30 ++++++++++++++++++++++++++++++
setup.py | 3 ++-
4 files changed, 63 insertions(+), 4 deletions(-)
diff --git a/bob/pad/base/script/figure.py b/bob/pad/base/script/figure.py
index a90a8a9..25fb027 100644
--- a/bob/pad/base/script/figure.py
+++ b/bob/pad/base/script/figure.py
@@ -93,6 +93,34 @@ class Metrics(measure_figure.Metrics):
)
+class MetricsVuln(measure_figure.Metrics):
+ def __init__(self, ctx, scores, evaluation, func_load):
+ super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)
+
+ ''' Compute metrics from score files'''
+ def compute(self, idx, input_scores, input_names):
+ ''' Compute metrics for the given criteria'''
+ neg_list, pos_list, _ = get_fta_list(input_scores)
+ dev_neg, dev_pos = neg_list[0], pos_list[0]
+ criter = self._criterion or 'eer'
+ threshold = calc_threshold(criter, dev_neg, dev_pos) \
+ if self._thres is None else self._thres[idx]
+ far, frr = farfrr(neg_list[1], pos_list[1], threshold)
+ iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
+ title = self._legends[idx] if self._legends is not None else None
+ headers = ['' or title, '%s (threshold=%.2g)' %
+ (criter.upper(), threshold)]
+ rows = []
+ rows.append(['FMR (%)', '{:>5.1f}%'.format(100*far)])
+ rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr*100)])
+ rows.append(['HTER (%)', '{:>5.1f}%'.format(50*(far+frr))])
+ rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100*iapmr)])
+ click.echo(
+ tabulate(rows, headers, self._tablefmt),
+ file=self.log_file
+ )
+
+
class HistPad(measure_figure.Hist):
''' Histograms for PAD '''
diff --git a/bob/pad/base/script/histograms.py b/bob/pad/base/script/histograms.py
index 73e8706..a8561de 100644
--- a/bob/pad/base/script/histograms.py
+++ b/bob/pad/base/script/histograms.py
@@ -72,7 +72,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.style_option()
@verbosity_option()
@click.pass_context
-def vuln(ctx, scores, evaluation, **kwargs):
+def vuln_hist(ctx, scores, evaluation, **kwargs):
'''Vulnerability analysis distributions.
Plots the histogram of score distributions. You need to provide 4 score
@@ -99,10 +99,10 @@ def vuln(ctx, scores, evaluation, **kwargs):
Examples:
- $ bob pad vuln licit/scores-dev licit/scores-eval \
+ $ bob pad vuln_hist licit/scores-dev licit/scores-eval \
spoof/scores-dev spoof/scores-eval
- $ bob pad vuln {licit,spoof}/scores-{dev,eval}
+ $ bob pad vuln_hist {licit,spoof}/scores-{dev,eval}
'''
process = figure.HistVuln(ctx, scores, evaluation, load.split)
process.run()
diff --git a/bob/pad/base/script/metrics.py b/bob/pad/base/script/metrics.py
index 840f367..4bfadf8 100644
--- a/bob/pad/base/script/metrics.py
+++ b/bob/pad/base/script/metrics.py
@@ -51,3 +51,33 @@ def metrics(ctx, scores, evaluation, **kwargs):
"""
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
+
+@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
+@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
+@common_options.eval_option()
+@common_options.table_option()
+@common_options.criterion_option()
+@common_options.thresholds_option()
+@open_file_mode_option()
+@common_options.output_log_metric_option()
+@common_options.legends_option()
+@verbosity_option()
+@click.pass_context
+def vuln_metrics(ctx, scores, **kwargs):
+ """Generate table of metrics for vulnerability PAD
+
+ You need to provide 2 or 4 scores
+ files for each PAD system in this order:
+
+ \b
+ * licit development scores
+ * licit evaluation scores
+ * spoof development scores
+ * spoof evaluation scores
+
+
+ Examples:
+ $ bob pad vuln_metrics {licit,spoof}/scores-{dev,eval}
+ """
+ process = figure.MetricsVuln(ctx, scores, True, load.split)
+ process.run()
diff --git a/setup.py b/setup.py
index 78c5729..bba9e12 100644
--- a/setup.py
+++ b/setup.py
@@ -140,8 +140,9 @@ setup(
# bob pad scripts
'bob.pad.cli': [
'metrics = bob.pad.base.script.metrics:metrics',
+ 'vuln_metrics = bob.pad.base.script.metrics:vuln_metrics',
'hist = bob.pad.base.script.histograms:hist',
- 'vuln = bob.pad.base.script.histograms:vuln',
+ 'vuln_hist = bob.pad.base.script.histograms:vuln_hist',
'det = bob.pad.base.script.det:det',
'epc = bob.pad.base.script.epc:epc',
'epsc = bob.pad.base.script.epc:epsc',
--
GitLab