From 1358cfabc9a4542ab978250eff5183e214c99ca8 Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Mon, 23 Apr 2018 09:27:42 +0200
Subject: [PATCH] Modify code according to the changes in bob.measure
---
bob/pad/base/script/det.py | 6 +-
bob/pad/base/script/epc.py | 12 +--
bob/pad/base/script/figure.py | 150 ++++++++++++------------------
bob/pad/base/script/histograms.py | 8 +-
bob/pad/base/script/metrics.py | 4 +-
5 files changed, 71 insertions(+), 109 deletions(-)
diff --git a/bob/pad/base/script/det.py b/bob/pad/base/script/det.py
index 0992ba9..35e626f 100644
--- a/bob/pad/base/script/det.py
+++ b/bob/pad/base/script/det.py
@@ -6,10 +6,8 @@ from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
-FUNC_SPLIT = lambda x: load.load_files(x, load.split)
-
@click.command()
-@common_options.scores_argument(eval_mandatory=True, min_len=2, nargs=-1)
+@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.const_layout_option()
@@ -48,5 +46,5 @@ def det(ctx, scores, criteria, real_data, **kwargs):
$ bob pad det {licit,spoof}/scores-{dev,eval}
"""
- process = figure.Det(ctx, scores, True, FUNC_SPLIT, criteria, real_data)
+ process = figure.Det(ctx, scores, True, load.split, criteria, real_data)
process.run()
diff --git a/bob/pad/base/script/epc.py b/bob/pad/base/script/epc.py
index 1f341dd..10b89f1 100644
--- a/bob/pad/base/script/epc.py
+++ b/bob/pad/base/script/epc.py
@@ -6,10 +6,8 @@ from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
-FUNC_SPLIT = lambda x: load.load_files(x, load.split)
-
@click.command()
-@common_options.scores_argument(eval_mandatory=True, min_len=2, nargs=-1)
+@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.const_layout_option()
@@ -43,11 +41,11 @@ def epc(ctx, scores, **kwargs):
$ bob pad epc {licit,spoof}/scores-{dev,eval}
"""
- process = figure.Epc(ctx, scores, True, FUNC_SPLIT)
+ process = figure.Epc(ctx, scores, True, load.split)
process.run()
@click.command()
-@common_options.scores_argument(eval_mandatory=True, min_len=2, nargs=-1)
+@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epsc.pdf')
@common_options.titles_option()
@common_options.const_layout_option()
@@ -102,12 +100,12 @@ def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, **kwargs):
if (ctx.meta['wer'] and ctx.meta['iapmr']):
raise click.BadParameter('Cannot plot both WER and IAPMR in 3D')
process = figure.Epsc3D(
- ctx, scores, True, FUNC_SPLIT,
+ ctx, scores, True, load.split,
criteria, var_param, fixed_param
)
else:
process = figure.Epsc(
- ctx, scores, True, FUNC_SPLIT,
+ ctx, scores, True, load.split,
criteria, var_param, fixed_param
)
process.run()
diff --git a/bob/pad/base/script/figure.py b/bob/pad/base/script/figure.py
index 078a6d6..9de35c8 100644
--- a/bob/pad/base/script/figure.py
+++ b/bob/pad/base/script/figure.py
@@ -8,7 +8,7 @@ import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
from tabulate import tabulate
from bob.extension.scripts.click_helper import verbosity_option
-from bob.measure.utils import (get_fta, get_thres)
+from bob.measure.utils import (get_fta, get_fta_list, get_thres)
from bob.measure import (
far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
)
@@ -57,15 +57,18 @@ class Metrics(measure_figure.Metrics):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
- def compute(self, idx, dev_score, dev_file=None,
- eval_score=None, eval_file=None):
+ def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
- dev_neg, dev_pos, _, eval_neg, eval_pos, _ =\
- self._process_scores(dev_score, eval_score)
+ neg_list, pos_list, _ = get_fta_list(input_scores)
+ dev_neg, dev_pos = neg_list[0], pos_list[0]
+ dev_file = input_names[0]
+ if self._eval:
+ eval_neg, eval_pos = neg_list[1], pos_list[1]
+ eval_file = input_names[1]
title = self._titles[idx] if self._titles is not None else None
headers = ['' or title, 'Development %s' % dev_file]
- if self._eval and eval_score is not None:
+ if self._eval:
headers.append('Eval. % s' % eval_file)
for m in ALL_CRITERIA:
raws = []
@@ -93,11 +96,11 @@ class HistPad(measure_figure.Hist):
def _setup_hist(self, neg, pos):
self._title_base = 'PAD'
self._density_hist(
- pos, label='Bona Fide', color='C1', **self._kwargs
+ pos[0], label='Bona Fide', color='C1'
)
self._density_hist(
- neg, label='Presentation attack', alpha=0.4, color='C7',
- hatch='\\\\', **self._kwargs
+ neg[0], label='Presentation attack', alpha=0.4, color='C7',
+ hatch='\\\\'
)
def _calc_pass_rate(threshold, scores):
@@ -134,48 +137,24 @@ def _iapmr_line_plot(scores, n_points=100, **kwargs):
mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)
-
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
_iapmr_dot(threshold, iapmr, real_data, **kwargs)
_iapmr_line_plot(scores, n_points=100, **kwargs)
-
class HistVuln(measure_figure.Hist):
''' Histograms for vulnerability '''
- def _get_neg_pos_thres(self, idx, dev_score, eval_score):
- assert len(dev_score) == self._min_arg
- dev_neg_list = []
- eval_neg_list = []
- dev_pos_list = []
- eval_pos_list = []
- for i in range(self._min_arg):
- dev_neg, dev_pos, _, eval_neg, eval_pos, _ = self._process_scores(
- dev_score[i], eval_score[i]
- )
- dev_neg_list.append(dev_neg)
- dev_pos_list.append(dev_pos)
- eval_neg_list.append(eval_neg)
- eval_pos_list.append(eval_pos)
-
- threshold = get_thres(
- self._criter, dev_neg_list[0], dev_pos_list[0]
- ) if self._thres is None else self._thres[idx]
- return (dev_neg_list, dev_pos_list,
- eval_neg_list, eval_pos_list, threshold)
def _setup_hist(self, neg, pos):
self._title_base = 'Vulnerability'
- assert len(neg) == len(pos) == self._min_arg
self._density_hist(
- pos[0], label='Bona Fide', color='C1', **self._kwargs
+ pos[0], label='Bona Fide', color='C1'
)
self._density_hist(
- neg[0], label='Zero-effort impostors', alpha=0.8, color='C0',
- **self._kwargs
+ neg[0], label='Zero-effort impostors', alpha=0.8, color='C0'
)
self._density_hist(
neg[1], label='Presentation attack', alpha=0.4, color='C7',
- hatch='\\\\', **self._kwargs
+ hatch='\\\\'
)
def _lines(self, threshold, neg, pos, **kwargs):
@@ -210,24 +189,6 @@ class PadPlot(measure_figure.PlotBase):
super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
mpl.rcParams['figure.constrained_layout.use'] = self._clayout
- def _process_scores(self, dev_score, eval_score):
- '''Process score files and return neg/pos/fta for eval and dev'''
- assert len(dev_score) == self._min_arg
- dev_neg_list = []
- eval_neg_list = []
- dev_pos_list = []
- eval_pos_list = []
- for i in range(self._min_arg):
- dev_neg, dev_pos, _, eval_neg, eval_pos, _ = \
- super(PadPlot, self)._process_scores(
- dev_score[i], eval_score[i]
- )
- dev_neg_list.append(dev_neg)
- dev_pos_list.append(dev_pos)
- eval_neg_list.append(eval_neg)
- eval_pos_list.append(eval_pos)
- return (dev_neg_list, dev_pos_list, None,
- eval_neg_list, eval_pos_list, None)
def end_process(self):
'''Close pdf '''
@@ -260,14 +221,19 @@ class Epc(PadPlot):
self._eval = True #always eval data with EPC
self._split = False
self._nb_figs = 1
+ self._title = ''
- def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None):
+ if self._min_arg != 4:
+ raise click.BadParameter("You must provide 4 scores files:{licit,"
+ "spoof}/{dev,eval}")
+
+ def compute(self, idx, input_scores, input_names):
''' Plot EPC for PAD'''
- licit_dev_neg = dev_score[0][0]
- licit_dev_pos = dev_score[0][1]
- licit_eval_neg = eval_score[0][0]
- licit_eval_pos = eval_score[0][1]
- spoof_eval_neg = eval_score[1][0]
+ licit_dev_neg = input_scores[0][0]
+ licit_dev_pos = input_scores[0][1]
+ licit_eval_neg = input_scores[1][0]
+ licit_eval_pos = input_scores[1][1]
+ spoof_eval_neg = input_scores[3][0]
mpl.gcf().clear()
epc_baseline = epc(
licit_dev_neg, licit_dev_pos, licit_eval_neg,
@@ -277,7 +243,7 @@ class Epc(PadPlot):
epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]],
color='C0',
label=self._label(
- 'WER', '%s-%s' % (dev_file[0], eval_file[0]), idx
+ 'WER', '%s-%s' % (input_names[0], input_names[1]), idx
),
linestyle='-'
)
@@ -299,7 +265,7 @@ class Epc(PadPlot):
color='C3',
linestyle='-',
label=self._label(
- 'IAPMR', '%s-%s' % (dev_file[0], eval_file[0]), idx
+ 'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
)
)
prob_ax.set_yticklabels(prob_ax.get_yticks())
@@ -330,21 +296,25 @@ class Epsc(PadPlot):
self._criteria = 'eer' if criteria is None else criteria
self._var_param = "omega" if var_param is None else var_param
self._fixed_param = 0.5 if fixed_param is None else fixed_param
- self._title = ''
self._eval = True #always eval data with EPC
self._split = False
self._nb_figs = 1
+ self._title = ''
+
+ if self._min_arg != 4:
+ raise click.BadParameter("You must provide 4 scores files:{licit,"
+ "spoof}/{dev,eval}")
- def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None):
+ def compute(self, idx, input_scores, input_names):
''' Plot EPSC for PAD'''
- licit_dev_neg = dev_score[0][0]
- licit_dev_pos = dev_score[0][1]
- licit_eval_neg = eval_score[0][0]
- licit_eval_pos = eval_score[0][1]
- spoof_dev_neg = dev_score[1][0]
- spoof_dev_pos = dev_score[1][1]
- spoof_eval_neg = eval_score[1][0]
- spoof_eval_pos = eval_score[1][1]
+ licit_dev_neg = input_scores[0][0]
+ licit_dev_pos = input_scores[0][1]
+ licit_eval_neg = input_scores[1][0]
+ licit_eval_pos = input_scores[1][1]
+ spoof_dev_neg = input_scores[2][0]
+ spoof_dev_pos = input_scores[2][1]
+ spoof_eval_neg = input_scores[3][0]
+ spoof_eval_pos = input_scores[3][1]
title = self._titles[idx] if self._titles is not None else None
mpl.gcf().clear()
@@ -443,16 +413,16 @@ class Epsc(PadPlot):
class Epsc3D(Epsc):
''' 3D EPSC plots for PAD'''
- def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None):
+ def compute(self, idx, input_scores, input_names):
''' Implements plots'''
- licit_dev_neg = dev_score[0][0]
- licit_dev_pos = dev_score[0][1]
- licit_eval_neg = eval_score[0][0]
- licit_eval_pos = eval_score[0][1]
- spoof_dev_neg = dev_score[1][0]
- spoof_dev_pos = dev_score[1][1]
- spoof_eval_neg = eval_score[1][0]
- spoof_eval_pos = eval_score[1][1]
+ licit_dev_neg = input_scores[0][0]
+ licit_dev_pos = input_scores[0][1]
+ licit_eval_neg = input_scores[1][0]
+ licit_eval_pos = input_scores[1][1]
+ spoof_dev_neg = input_scores[2][0]
+ spoof_dev_pos = input_scores[2][1]
+ spoof_eval_neg = input_scores[3][0]
+ spoof_eval_pos = input_scores[3][1]
title = self._titles[idx] if self._titles is not None else None
@@ -516,14 +486,14 @@ class Det(PadPlot):
self._criteria = criteria
self._real_data = True if real_data is None else real_data
- def compute(self, idx, dev_score, dev_file, eval_score, eval_file=None):
+ def compute(self, idx, input_scores, input_names):
''' Implements plots'''
- licit_dev_neg = dev_score[0][0]
- licit_dev_pos = dev_score[0][1]
- licit_eval_neg = eval_score[0][0]
- licit_eval_pos = eval_score[0][1]
- spoof_eval_neg = eval_score[1][0] if len(dev_score) > 1 else None
- spoof_eval_pos = eval_score[1][1] if len(dev_score) > 1 else None
+ licit_dev_neg = input_scores[0][0]
+ licit_dev_pos = input_scores[0][1]
+ licit_eval_neg = input_scores[1][0]
+ licit_eval_pos = input_scores[1][1]
+ spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
+ spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
det(
licit_eval_neg,
@@ -531,7 +501,7 @@ class Det(PadPlot):
self._points,
color=self._colors[idx],
linestyle='-',
- label=self._label("licit", dev_file[0], idx)
+ label=self._label("licit", input_names[0], idx)
)
if not self._no_spoof and spoof_eval_neg is not None:
det(
@@ -540,7 +510,7 @@ class Det(PadPlot):
self._points,
color=self._colors[idx],
linestyle='--',
- label=self._label("spoof", eval_file[0], idx)
+ label=self._label("spoof", input_names[3], idx)
)
if self._criteria is None:
diff --git a/bob/pad/base/script/histograms.py b/bob/pad/base/script/histograms.py
index e342718..4e83d6a 100644
--- a/bob/pad/base/script/histograms.py
+++ b/bob/pad/base/script/histograms.py
@@ -6,8 +6,6 @@ from bob.extension.scripts.click_helper import (verbosity_option, bool_option)
from bob.bio.base.score import load
from . import figure
-FUNC_SPLIT = lambda x: load.load_files(x, load.split)
-
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@@ -44,11 +42,11 @@ def hist(ctx, scores, evaluation, **kwargs):
$ bob pad hist --criter hter dev-scores1 eval-scores1
"""
- process = figure.HistPad(ctx, scores, evaluation, FUNC_SPLIT)
+ process = figure.HistPad(ctx, scores, evaluation, load.split)
process.run()
@click.command()
-@common_options.scores_argument(nargs=-1, eval_mandatory=True, min_len=2)
+@common_options.scores_argument(nargs=-1, min_arg=2)
@common_options.output_plot_file_option(default_out='vuln.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@@ -102,5 +100,5 @@ def vuln(ctx, scores, evaluation, **kwargs):
$ bob pad vuln {licit,spoof}/scores-{dev,eval}
'''
- process = figure.HistVuln(ctx, scores, evaluation, FUNC_SPLIT)
+ process = figure.HistVuln(ctx, scores, evaluation, load.split)
process.run()
diff --git a/bob/pad/base/script/metrics.py b/bob/pad/base/script/metrics.py
index 151468f..60db789 100644
--- a/bob/pad/base/script/metrics.py
+++ b/bob/pad/base/script/metrics.py
@@ -7,8 +7,6 @@ from bob.extension.scripts.click_helper import (verbosity_option,
from bob.bio.base.score import load
from . import figure
-FUNC_SPLIT = lambda x: load.load_files(x, load.split)
-
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
@@ -51,5 +49,5 @@ def metrics(ctx, scores, evaluation, **kwargs):
$ bob pad metrics /path/to/system{1,2,3}/score-{dev,eval}
"""
- process = figure.Metrics(ctx, scores, evaluation, FUNC_SPLIT)
+ process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
--
GitLab