Commit cd1fedfe authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Remove bob vuln metrics and evaluate commands

since they were not well defined and we do
not know what should be in there.

rename --hlines-at to --fnmr in bob vuln roc,det commands

small nit-pick fixes overall
parent 861911a5
Pipeline #21368 passed with stage
in 26 minutes and 19 seconds
......@@ -88,7 +88,7 @@ def evaluate(ctx, scores, evaluation, **kwargs):
common_options.MULTI_METRICS_HELP.format(
names='FtA, APCER, BPCER, FAR, FRR, ACER',
criteria=CRITERIA, score_format=SCORE_FORMAT,
command='bob measure multi-metrics'),
command='bob pad multi-metrics'),
criteria=CRITERIA)
def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)
......
This diff is collapsed.
'''Runs error analysis on score sets, outputs metrics and plots'''
import math
import click
import numpy as np
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
import bob.bio.base.script.figure as bio_figure
from tabulate import tabulate
from bob.measure.utils import get_fta_list
from bob.measure import (
frr_threshold, far_threshold, eer_threshold, min_hter_threshold, farfrr,
epc, ppndf, min_weighted_error_rate_threshold
frr_threshold, far_threshold, farfrr,
ppndf, min_weighted_error_rate_threshold
)
from bob.measure.plot import (det, det_axis, roc_for_far, log_values, epc)
from bob.measure import plot
from . import error_utils
class Metrics(measure_figure.Metrics):
def __init__(self, ctx, scores, evaluation, func_load):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
# extract pos and negative and remove NaNs
neg_list, pos_list, _ = get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
criter = self._criterion or 'eer'
threshold = error_utils.calc_threshold(criter, dev_neg, dev_pos) \
if self._thres is None else self._thres[idx]
far, frr = farfrr(neg_list[1], pos_list[1], threshold)
iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
title = self._legends[idx] if self._legends is not None else None
headers = ['' or title, '%s (threshold=%.2g)' %
(criter.upper(), threshold)]
rows = []
rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
rows.append(['FNMR (%)', '{:>5.1f}%'.format(frr * 100)])
rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
click.echo(
tabulate(rows, headers, self._tablefmt),
file=self.log_file
)
def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
# plot a dot on threshold versus IAPMR line and show IAPMR as a number
axlim = mpl.axis()
......@@ -185,7 +152,7 @@ class Epc(PadPlot):
mpl.gcf().clear()
mpl.grid()
epc(
plot.epc(
licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos,
self._points,
color='C0', linestyle=self._linestyles[idx],
......@@ -228,7 +195,6 @@ class Epc(PadPlot):
ax1.tick_params(axis='y', colors='C0')
ax1.spines['left'].set_color('C0')
title = self._legends[idx] if self._legends is not None else self._title
if title.replace(' ', ''):
mpl.title(title)
......@@ -448,7 +414,7 @@ class BaseVulnDetRoc(PadPlot):
super(BaseVulnDetRoc, self).__init__(
ctx, scores, evaluation, func_load)
self._no_spoof = no_spoof
self._hlines_at = ctx.meta.get('hlines_at', [])
self._fnmrs_at = ctx.meta.get('fnmr', [])
self._real_data = True if real_data is None else real_data
self._legend_loc = None
......@@ -469,7 +435,7 @@ class BaseVulnDetRoc(PadPlot):
if not self._no_spoof and spoof_neg is not None:
ax1 = mpl.gca()
ax2 = ax1.twiny()
ax2.set_xlabel('IAPMR', color='C3')
ax2.set_xlabel('IAPMR (%)', color='C3')
ax2.set_xticklabels(ax2.get_xticks())
ax2.tick_params(axis='x', colors='C3')
ax2.xaxis.label.set_color('C3')
......@@ -487,10 +453,10 @@ class BaseVulnDetRoc(PadPlot):
)
mpl.sca(ax1)
if self._hlines_at is None:
if self._fnmrs_at is None:
return
for line in self._hlines_at:
for line in self._fnmrs_at:
thres_baseline = frr_threshold(licit_neg, licit_pos, line)
axlim = mpl.axis()
......@@ -547,7 +513,6 @@ class BaseVulnDetRoc(PadPlot):
label=label_spoof
) # FAR point, spoof scenario
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
close pdf is needed '''
......@@ -594,9 +559,9 @@ class DetVuln(BaseVulnDetRoc):
def __init__(self, ctx, scores, evaluation, func_load, real_data,
no_spoof):
super(DetVuln, self).__init__(ctx, scores, evaluation, func_load,
real_data, no_spoof)
self._x_label = self._x_label or "FMR"
self._y_label = self._y_label or "FNMR"
real_data, no_spoof)
self._x_label = self._x_label or "FMR (%)"
self._y_label = self._y_label or "FNMR (%)"
add = ''
if not self._no_spoof:
add = " and overlaid SPOOF scenario"
......@@ -605,16 +570,16 @@ class DetVuln(BaseVulnDetRoc):
def _set_axis(self):
if self._axlim is not None and None not in self._axlim:
det_axis(self._axlim)
plot.det_axis(self._axlim)
else:
det_axis([0.01, 99, 0.01, 99])
plot.det_axis([0.01, 99, 0.01, 99])
def _get_farfrr(self, x, y, thres):
points = farfrr(x, y, thres)
return points, [ppndf(i) for i in points]
def _plot(self, x, y, points, **kwargs):
det(
plot.det(
x, y, points,
color=kwargs.get('color'),
linestyle=kwargs.get('linestyle'),
......@@ -639,9 +604,9 @@ class RocVuln(BaseVulnDetRoc):
self._legend_loc = self._legend_loc or best_legend
def _plot(self, x, y, points, **kwargs):
roc_for_far(
plot.roc_for_far(
x, y,
far_values=log_values(self._min_dig or -4),
far_values=plot.log_values(self._min_dig or -4),
CAR=self._semilogx,
color=kwargs.get('color'), linestyle=kwargs.get('linestyle'),
label=kwargs.get('label')
......@@ -694,8 +659,8 @@ class FmrIapmr(PadPlot):
title = self._title if self._title is not None else "FMR vs IAPMR"
if title.replace(' ', ''):
mpl.title(title)
mpl.xlabel(self._x_label or "FMR (%)")
mpl.ylabel(self._y_label or "IAPMR (%)")
mpl.xlabel(self._x_label or "FMR")
mpl.ylabel(self._y_label or "IAPMR")
mpl.grid(True, color=self._grid_color)
if self._disp_legend:
mpl.legend(loc=self._legend_loc)
......
import sys
import click
from click.testing import CliRunner
import pkg_resources
from ..script import (pad_commands, vuln_commands)
def test_det_pad():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -19,7 +16,6 @@ def test_det_pad():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_det_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -31,7 +27,7 @@ def test_det_vuln():
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(vuln_commands.det, ['-hla', '0.2',
result = runner.invoke(vuln_commands.det, ['-fnmr', '0.2',
'-o',
'DET.pdf',
licit_dev, licit_test,
......@@ -39,7 +35,6 @@ def test_det_vuln():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_fmr_iapmr_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -64,7 +59,6 @@ def test_fmr_iapmr_vuln():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist_pad():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -96,7 +90,6 @@ def test_hist_pad():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -123,28 +116,6 @@ def test_hist_vuln():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_metrics_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.metrics,
['--criterion', 'eer', licit_dev, licit_test,
spoof_dev, spoof_test]
)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_metrics_pad():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -159,7 +130,6 @@ def test_metrics_pad():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
......@@ -199,7 +169,7 @@ def test_epsc_vuln():
['--output', 'epsc.pdf',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output)
assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
result = runner.invoke(vuln_commands.epsc,
['--output', 'epsc.pdf', '-I',
......@@ -221,8 +191,7 @@ def test_epsc_vuln():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate_vuln():
def test_evaluate_pad():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
......@@ -236,20 +205,3 @@ def test_evaluate_vuln():
result = runner.invoke(pad_commands.evaluate,
[licit_dev, licit_test, spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(vuln_commands.evaluate,
[licit_dev, licit_test, spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -151,7 +151,6 @@ setup(
# bob vuln scripts
'bob.vuln.cli': [
'metrics = bob.pad.base.script.vuln_commands:metrics',
'hist = bob.pad.base.script.vuln_commands:hist',
'det = bob.pad.base.script.vuln_commands:det',
'roc = bob.pad.base.script.vuln_commands:roc',
......@@ -159,7 +158,6 @@ setup(
'epsc = bob.pad.base.script.vuln_commands:epsc',
'gen = bob.pad.base.script.vuln_commands:gen',
'fmr_iapmr = bob.pad.base.script.vuln_commands:fmr_iapmr',
'evaluate = bob.pad.base.script.vuln_commands:evaluate',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment