Commit eee7f963 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

[script][figure,vuln_commands] Only use licit and spoof for det and roc.

Horizontal line for FNMR
parent b503f3e6
Pipeline #21209 failed with stage
in 19 minutes and 26 seconds
......@@ -12,7 +12,7 @@ from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option,
bool_option,
AliasedGroup)
AliasedGroup, list_float_option)
from bob.core import random
from bob.io.base import create_directories_safe
from bob.bio.base.score import load
......@@ -23,6 +23,15 @@ NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000
def hlines_at_option(dflt=' ', **kwargs):
'''Get option to draw const FNMRlines'''
return list_float_option(
name='hlines-at', short_name='hla',
desc='If given, draw horizontal lines at the given axis positions. '
'Your values must be separated with a comma (,) without space. '
'This option works in ROC and DET curves.',
nitems=None, dflt=dflt, **kwargs
)
def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
......@@ -101,7 +110,7 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_roc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
......@@ -116,37 +125,32 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('-c', '--criteria', default=None, show_default=True,
help='Criteria for threshold selection',
type=click.Choice(('eer', 'min-hter')))
@click.option('--real-data/--no-real-data', default=True, show_default=True,
help='If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.')
@hlines_at_option()
@click.pass_context
def roc(ctx, scores, criteria, real_data, **kwargs):
def roc(ctx, scores, real_data, **kwargs):
"""Plot ROC
You need to provide 4 scores
You need to provide 2 scores
files for each vulnerability system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
* licit scores
* spoof scores
Examples:
$ bob vuln roc -v dev-scores eval-scores
$ bob vuln roc -v licit-scores spoof-scores
$ bob vuln roc -v {licit,spoof}/scores-{dev,eval}
$ bob vuln roc -v scores-{licit,spoof}
"""
process = figure.RocVuln(ctx, scores, True, load.split, criteria, real_data,
False)
process = figure.RocVuln(ctx, scores, True, load.split, real_data, False)
process.run()
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_det.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
......@@ -160,32 +164,28 @@ def roc(ctx, scores, criteria, real_data, **kwargs):
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('-c', '--criteria', default=None, show_default=True,
help='Criteria for threshold selection',
type=click.Choice(('eer', 'min-hter')))
@click.option('--real-data/--no-real-data', default=True, show_default=True,
help='If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.')
@hlines_at_option()
@click.pass_context
def det(ctx, scores, criteria, real_data, **kwargs):
def det(ctx, scores, real_data, **kwargs):
"""Plot DET
You need to provide 4 scores
files for each vuln system in this order:
You need to provide 2 scores
files for each vulnerability system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
* licit scores
* spoof scores
Examples:
$ bob vuln det -v dev-scores eval-scores
$ bob vuln det -v licit-scores spoof-scores
$ bob vuln det -v {licit,spoof}/scores-{dev,eval}
$ bob vuln det -v scores-{licit,spoof}
"""
process = figure.Det(ctx, scores, True, load.split, criteria, real_data,
False)
process = figure.Det(ctx, scores, True, load.split, real_data, False)
process.run()
......
'''Runs error analysis on score sets, outputs metrics and plots'''
import math
import click
import numpy as np
import matplotlib.pyplot as mpl
......@@ -434,32 +435,30 @@ class Epsc3D(Epsc):
class BaseVulnDetRoc(PadPlot):
'''Base for DET and ROC'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
def __init__(self, ctx, scores, evaluation, func_load, real_data,
no_spoof):
super(BaseVulnDetRoc, self).__init__(
ctx, scores, evaluation, func_load)
self._no_spoof = no_spoof
self._criteria = criteria or 'eer'
self._hlines_at = ctx.meta.get('hlines_at', [])
self._real_data = True if real_data is None else real_data
self._legend_loc = None
def compute(self, idx, input_scores, input_names):
''' Implements plots'''
licit_dev_neg = input_scores[0][0]
licit_dev_pos = input_scores[0][1]
licit_eval_neg = input_scores[1][0]
licit_eval_pos = input_scores[1][1]
spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
licit_neg = input_scores[0][0]
licit_pos = input_scores[0][1]
spoof_neg = input_scores[1][0]
spoof_pos = input_scores[1][1]
self._plot(
licit_eval_neg,
licit_eval_pos,
licit_neg,
licit_pos,
self._points,
color='C0',
linestyle='-',
label=self._label("licit", input_names[0], idx)
)
if not self._no_spoof and spoof_eval_neg is not None:
if not self._no_spoof and spoof_neg is not None:
ax1 = mpl.gca()
ax2 = ax1.twiny()
ax2.set_xlabel('IAPMR', color='C3')
......@@ -467,114 +466,117 @@ class BaseVulnDetRoc(PadPlot):
ax2.tick_params(axis='x', colors='C3')
ax2.xaxis.label.set_color('C3')
ax2.spines['top'].set_color('C3')
ax2.spines['bottom'].set_color('C0')
ax1.xaxis.label.set_color('C0')
ax1.tick_params(axis='x', colors='C0')
self._plot(
spoof_eval_neg,
spoof_eval_pos,
spoof_neg,
spoof_pos,
self._points,
color='C3',
linestyle=':',
label=self._label("spoof", input_names[3], idx)
label=self._label("spoof", input_names[1], idx)
)
mpl.sca(ax1)
if self._criteria is None or self._no_spoof:
if self._hlines_at is None:
return
thres_baseline = calc_threshold(
self._criteria, licit_dev_neg, licit_dev_pos
)
for line in self._hlines_at:
print (line)
thres_baseline = frr_threshold(licit_neg, licit_pos, line)
axlim = mpl.axis()
axlim = mpl.axis()
farfrr_licit, farfrr_licit_det = self._get_farfrr(
licit_eval_neg, licit_eval_pos,
thres_baseline
)
if farfrr_licit is None:
return
farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
spoof_eval_neg, spoof_eval_pos,
frr_threshold(spoof_eval_neg, spoof_eval_pos,
farfrr_licit[1])
)
farfrr_licit, farfrr_licit_det = self._get_farfrr(
licit_neg, licit_pos,
thres_baseline
)
if farfrr_licit is None:
return
if not self._real_data:
mpl.axhline(
y=farfrr_licit_det[1],
xmin=axlim[2],
xmax=axlim[3],
color='k',
linestyle='--',
label="%s @ EER" % self._y_label)
else:
mpl.axhline(
y=farfrr_licit_det[1],
xmin=axlim[0],
xmax=axlim[1],
color='k',
linestyle='--',
label="%s = %.2f%%" %
(self._y_label, farfrr_licit[1] * 100))
farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
spoof_neg, spoof_pos,
frr_threshold(spoof_neg, spoof_pos, farfrr_licit[1])
)
mpl.plot(
farfrr_licit_det[0],
farfrr_licit_det[1],
'o',
color='C0',
) # FAR point, licit scenario
mpl.plot(
farfrr_spoof_det[0],
farfrr_spoof_det[1],
'o',
color='C3',
) # FAR point, spoof scenario
# annotate the FAR points
if farfrr_licit_det[0] < farfrr_spoof_det[0]:
xyannotate_licit = [
farfrr_licit_det[0] - 0.7,
farfrr_licit_det[1] - 0.4,
]
xyannotate_spoof = [
0.1 + farfrr_spoof_det[0],
farfrr_spoof_det[1] + 0.3,
]
else:
xyannotate_spoof = [
farfrr_licit_det[0] - 0.7,
farfrr_licit_det[1] - 0.4,
]
xyannotate_licit = [
0.1 + farfrr_spoof_det[0],
farfrr_spoof_det[1] + 0.3,
]
if not self._real_data:
mpl.annotate(
'%s @ operating point' % self._y_label,
xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
xycoords='data',
xytext=(xyannotate_licit[0], xyannotate_licit[1]))
mpl.annotate(
'IAPMR @ operating point',
xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
xycoords='data',
xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
else:
mpl.annotate(
'FMR=%.2f%%' % (farfrr_licit[0] * 100),
xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
xycoords='data',
xytext=(xyannotate_licit[0], xyannotate_licit[1]),
if not self._real_data:
mpl.axhline(
y=farfrr_licit_det[1],
xmin=axlim[2],
xmax=axlim[3],
color='k',
linestyle='--',
label="%s @ EER" % self._y_label)
else:
mpl.axhline(
y=farfrr_licit_det[1],
xmin=axlim[0],
xmax=axlim[1],
color='k',
linestyle='--',
label="%s = %.2f%%" %
(self._y_label, farfrr_licit[1] * 100))
mpl.plot(
farfrr_licit_det[0],
farfrr_licit_det[1],
'o',
color='C0',
size='small')
mpl.annotate(
'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
xycoords='data',
xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
) # FAR point, licit scenario
mpl.plot(
farfrr_spoof_det[0],
farfrr_spoof_det[1],
'o',
color='C3',
size='small')
) # FAR point, spoof scenario
# annotate the FAR points
if farfrr_licit_det[0] < farfrr_spoof_det[0]:
xyannotate_licit = [
farfrr_licit_det[0] - 0.7,
farfrr_licit_det[1] - 0.4,
]
xyannotate_spoof = [
0.1 + farfrr_spoof_det[0],
farfrr_spoof_det[1] + 0.3,
]
else:
xyannotate_spoof = [
farfrr_licit_det[0] - 0.7,
farfrr_licit_det[1] - 0.4,
]
xyannotate_licit = [
0.1 + farfrr_spoof_det[0],
farfrr_spoof_det[1] + 0.3,
]
if not self._real_data:
mpl.annotate(
'%s @ operating point' % self._y_label,
xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
xycoords='data',
xytext=(xyannotate_licit[0], xyannotate_licit[1]))
mpl.annotate(
'IAPMR @ operating point',
xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
xycoords='data',
xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
else:
mpl.annotate(
'FMR=%.2f%%' % (farfrr_licit[0] * 100),
xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
xycoords='data',
xytext=(xyannotate_licit[0], xyannotate_licit[1]),
color='C0',
size='small')
mpl.annotate(
'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
xycoords='data',
xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
color='C3',
size='small')
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
......@@ -619,10 +621,10 @@ class BaseVulnDetRoc(PadPlot):
class DetVuln(BaseVulnDetRoc):
'''DET for vuln'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
def __init__(self, ctx, scores, evaluation, func_load, real_data,
no_spoof):
super(Det, self).__init__(ctx, scores, evaluation, func_load,
criteria, real_data, no_spoof)
real_data, no_spoof)
self._x_label = self._x_label or "FMR"
self._y_label = self._y_label or "FNMR"
add = ''
......@@ -653,10 +655,9 @@ class DetVuln(BaseVulnDetRoc):
class RocVuln(BaseVulnDetRoc):
'''ROC for vuln'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
no_spoof):
def __init__(self, ctx, scores, evaluation, func_load, real_data, no_spoof):
super(RocVuln, self).__init__(ctx, scores, evaluation, func_load,
criteria, real_data, no_spoof)
real_data, no_spoof)
self._x_label = self._x_label or "FMR"
self._y_label = self._y_label or "1 - FNMR"
self._semilogx = ctx.meta.get('semilogx', True)
......@@ -677,6 +678,7 @@ class RocVuln(BaseVulnDetRoc):
)
class FmrIapmr(PadPlot):
'''FMR vs IAPMR'''
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment