Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pad.base
Commits
eee7f963
Commit
eee7f963
authored
Jun 20, 2018
by
Theophile GENTILHOMME
Browse files
[script][figure,vuln_commands] Only use licit and spoof for det and roc.
Horizontal line for FNMR
parent
b503f3e6
Pipeline
#21209
failed with stage
in 19 minutes and 26 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/pad/base/script/vuln_commands.py
View file @
eee7f963
...
...
@@ -12,7 +12,7 @@ from bob.measure.script import common_options
from
bob.extension.scripts.click_helper
import
(
verbosity_option
,
open_file_mode_option
,
bool_option
,
AliasedGroup
)
AliasedGroup
,
list_float_option
)
from
bob.core
import
random
from
bob.io.base
import
create_directories_safe
from
bob.bio.base.score
import
load
...
...
@@ -23,6 +23,15 @@ NUM_ZEIMPOSTORS = 5000
NUM_PA
=
5000
def
hlines_at_option
(
dflt
=
' '
,
**
kwargs
):
'''Get option to draw const FNMRlines'''
return
list_float_option
(
name
=
'hlines-at'
,
short_name
=
'hla'
,
desc
=
'If given, draw horizontal lines at the given axis positions. '
'Your values must be separated with a comma (,) without space. '
'This option works in ROC and DET curves.'
,
nitems
=
None
,
dflt
=
dflt
,
**
kwargs
)
def
gen_score_distr
(
mean_gen
,
mean_zei
,
mean_pa
,
sigma_gen
=
1
,
sigma_zei
=
1
,
...
...
@@ -101,7 +110,7 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
@
click
.
command
()
@
common_options
.
scores_argument
(
min_arg
=
2
,
force_eval
=
True
,
nargs
=-
1
)
@
common_options
.
scores_argument
(
min_arg
=
2
,
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'vuln_roc.pdf'
)
@
common_options
.
legends_option
()
@
common_options
.
no_legend_option
()
...
...
@@ -116,37 +125,32 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
@
common_options
.
x_rotation_option
(
dflt
=
45
)
@
common_options
.
x_label_option
()
@
common_options
.
y_label_option
()
@
click
.
option
(
'-c'
,
'--criteria'
,
default
=
None
,
show_default
=
True
,
help
=
'Criteria for threshold selection'
,
type
=
click
.
Choice
((
'eer'
,
'min-hter'
)))
@
click
.
option
(
'--real-data/--no-real-data'
,
default
=
True
,
show_default
=
True
,
help
=
'If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.'
)
@
hlines_at_option
()
@
click
.
pass_context
def
roc
(
ctx
,
scores
,
criteria
,
real_data
,
**
kwargs
):
def
roc
(
ctx
,
scores
,
real_data
,
**
kwargs
):
"""Plot ROC
You need to provide
4
scores
You need to provide
2
scores
files for each vulnerability system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
* licit scores
* spoof scores
Examples:
$ bob vuln roc -v
dev
-scores
eval
-scores
$ bob vuln roc -v
licit
-scores
spoof
-scores
$ bob vuln roc -v {licit,spoof}
/scores-{dev,eval}
$ bob vuln roc -v
scores-
{licit,spoof}
"""
process
=
figure
.
RocVuln
(
ctx
,
scores
,
True
,
load
.
split
,
criteria
,
real_data
,
False
)
process
=
figure
.
RocVuln
(
ctx
,
scores
,
True
,
load
.
split
,
real_data
,
False
)
process
.
run
()
@
click
.
command
()
@
common_options
.
scores_argument
(
min_arg
=
2
,
force_eval
=
True
,
nargs
=-
1
)
@
common_options
.
scores_argument
(
min_arg
=
2
,
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'vuln_det.pdf'
)
@
common_options
.
legends_option
()
@
common_options
.
no_legend_option
()
...
...
@@ -160,32 +164,28 @@ def roc(ctx, scores, criteria, real_data, **kwargs):
@
common_options
.
x_rotation_option
(
dflt
=
45
)
@
common_options
.
x_label_option
()
@
common_options
.
y_label_option
()
@
click
.
option
(
'-c'
,
'--criteria'
,
default
=
None
,
show_default
=
True
,
help
=
'Criteria for threshold selection'
,
type
=
click
.
Choice
((
'eer'
,
'min-hter'
)))
@
click
.
option
(
'--real-data/--no-real-data'
,
default
=
True
,
show_default
=
True
,
help
=
'If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.'
)
@
hlines_at_option
()
@
click
.
pass_context
def
det
(
ctx
,
scores
,
criteria
,
real_data
,
**
kwargs
):
def
det
(
ctx
,
scores
,
real_data
,
**
kwargs
):
"""Plot DET
You need to provide 4 scores
files for each vuln system in this order:
You need to provide 2 scores
files for each vulnerability system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
* licit scores
* spoof scores
Examples:
$ bob vuln det -v
dev
-scores
eval
-scores
$ bob vuln det -v
licit
-scores
spoof
-scores
$ bob vuln det -v {licit,spoof}
/scores-{dev,eval}
$ bob vuln det -v
scores-
{licit,spoof}
"""
process
=
figure
.
Det
(
ctx
,
scores
,
True
,
load
.
split
,
criteria
,
real_data
,
False
)
process
=
figure
.
Det
(
ctx
,
scores
,
True
,
load
.
split
,
real_data
,
False
)
process
.
run
()
...
...
bob/pad/base/script/vuln_figure.py
View file @
eee7f963
'''Runs error analysis on score sets, outputs metrics and plots'''
import
math
import
click
import
numpy
as
np
import
matplotlib.pyplot
as
mpl
...
...
@@ -434,32 +435,30 @@ class Epsc3D(Epsc):
class
BaseVulnDetRoc
(
PadPlot
):
'''Base for DET and ROC'''
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
criteria
,
real_data
,
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
real_data
,
no_spoof
):
super
(
BaseVulnDetRoc
,
self
).
__init__
(
ctx
,
scores
,
evaluation
,
func_load
)
self
.
_no_spoof
=
no_spoof
self
.
_
criteria
=
criteria
or
'eer'
self
.
_
hlines_at
=
ctx
.
meta
.
get
(
'hlines_at'
,
[])
self
.
_real_data
=
True
if
real_data
is
None
else
real_data
self
.
_legend_loc
=
None
def
compute
(
self
,
idx
,
input_scores
,
input_names
):
''' Implements plots'''
licit_dev_neg
=
input_scores
[
0
][
0
]
licit_dev_pos
=
input_scores
[
0
][
1
]
licit_eval_neg
=
input_scores
[
1
][
0
]
licit_eval_pos
=
input_scores
[
1
][
1
]
spoof_eval_neg
=
input_scores
[
3
][
0
]
if
len
(
input_scores
)
>
2
else
None
spoof_eval_pos
=
input_scores
[
3
][
1
]
if
len
(
input_scores
)
>
2
else
None
licit_neg
=
input_scores
[
0
][
0
]
licit_pos
=
input_scores
[
0
][
1
]
spoof_neg
=
input_scores
[
1
][
0
]
spoof_pos
=
input_scores
[
1
][
1
]
self
.
_plot
(
licit_
eval_
neg
,
licit_
eval_
pos
,
licit_neg
,
licit_pos
,
self
.
_points
,
color
=
'C0'
,
linestyle
=
'-'
,
label
=
self
.
_label
(
"licit"
,
input_names
[
0
],
idx
)
)
if
not
self
.
_no_spoof
and
spoof_
eval_
neg
is
not
None
:
if
not
self
.
_no_spoof
and
spoof_neg
is
not
None
:
ax1
=
mpl
.
gca
()
ax2
=
ax1
.
twiny
()
ax2
.
set_xlabel
(
'IAPMR'
,
color
=
'C3'
)
...
...
@@ -467,114 +466,117 @@ class BaseVulnDetRoc(PadPlot):
ax2
.
tick_params
(
axis
=
'x'
,
colors
=
'C3'
)
ax2
.
xaxis
.
label
.
set_color
(
'C3'
)
ax2
.
spines
[
'top'
].
set_color
(
'C3'
)
ax2
.
spines
[
'bottom'
].
set_color
(
'C0'
)
ax1
.
xaxis
.
label
.
set_color
(
'C0'
)
ax1
.
tick_params
(
axis
=
'x'
,
colors
=
'C0'
)
self
.
_plot
(
spoof_
eval_
neg
,
spoof_
eval_
pos
,
spoof_neg
,
spoof_pos
,
self
.
_points
,
color
=
'C3'
,
linestyle
=
':'
,
label
=
self
.
_label
(
"spoof"
,
input_names
[
3
],
idx
)
label
=
self
.
_label
(
"spoof"
,
input_names
[
1
],
idx
)
)
mpl
.
sca
(
ax1
)
if
self
.
_
criteria
is
None
or
self
.
_no_spoof
:
if
self
.
_
hlines_at
is
None
:
return
thres_baseline
=
calc_threshold
(
self
.
_criteria
,
licit_dev_neg
,
licit_dev_pos
)
for
line
in
self
.
_hlines_at
:
print
(
line
)
thres_baseline
=
frr_threshold
(
licit_neg
,
licit_pos
,
line
)
axlim
=
mpl
.
axis
()
axlim
=
mpl
.
axis
()
farfrr_licit
,
farfrr_licit_det
=
self
.
_get_farfrr
(
licit_eval_neg
,
licit_eval_pos
,
thres_baseline
)
if
farfrr_licit
is
None
:
return
farfrr_spoof
,
farfrr_spoof_det
=
self
.
_get_farfrr
(
spoof_eval_neg
,
spoof_eval_pos
,
frr_threshold
(
spoof_eval_neg
,
spoof_eval_pos
,
farfrr_licit
[
1
])
)
farfrr_licit
,
farfrr_licit_det
=
self
.
_get_farfrr
(
licit_neg
,
licit_pos
,
thres_baseline
)
if
farfrr_licit
is
None
:
return
if
not
self
.
_real_data
:
mpl
.
axhline
(
y
=
farfrr_licit_det
[
1
],
xmin
=
axlim
[
2
],
xmax
=
axlim
[
3
],
color
=
'k'
,
linestyle
=
'--'
,
label
=
"%s @ EER"
%
self
.
_y_label
)
else
:
mpl
.
axhline
(
y
=
farfrr_licit_det
[
1
],
xmin
=
axlim
[
0
],
xmax
=
axlim
[
1
],
color
=
'k'
,
linestyle
=
'--'
,
label
=
"%s = %.2f%%"
%
(
self
.
_y_label
,
farfrr_licit
[
1
]
*
100
))
farfrr_spoof
,
farfrr_spoof_det
=
self
.
_get_farfrr
(
spoof_neg
,
spoof_pos
,
frr_threshold
(
spoof_neg
,
spoof_pos
,
farfrr_licit
[
1
])
)
mpl
.
plot
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
],
'o'
,
color
=
'C0'
,
)
# FAR point, licit scenario
mpl
.
plot
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
],
'o'
,
color
=
'C3'
,
)
# FAR point, spoof scenario
# annotate the FAR points
if
farfrr_licit_det
[
0
]
<
farfrr_spoof_det
[
0
]:
xyannotate_licit
=
[
farfrr_licit_det
[
0
]
-
0.7
,
farfrr_licit_det
[
1
]
-
0.4
,
]
xyannotate_spoof
=
[
0.1
+
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]
+
0.3
,
]
else
:
xyannotate_spoof
=
[
farfrr_licit_det
[
0
]
-
0.7
,
farfrr_licit_det
[
1
]
-
0.4
,
]
xyannotate_licit
=
[
0.1
+
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]
+
0.3
,
]
if
not
self
.
_real_data
:
mpl
.
annotate
(
'%s @ operating point'
%
self
.
_y_label
,
xy
=
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_licit
[
0
],
xyannotate_licit
[
1
]))
mpl
.
annotate
(
'IAPMR @ operating point'
,
xy
=
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_spoof
[
0
],
xyannotate_spoof
[
1
]))
else
:
mpl
.
annotate
(
'FMR=%.2f%%'
%
(
farfrr_licit
[
0
]
*
100
),
xy
=
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_licit
[
0
],
xyannotate_licit
[
1
]),
if
not
self
.
_real_data
:
mpl
.
axhline
(
y
=
farfrr_licit_det
[
1
],
xmin
=
axlim
[
2
],
xmax
=
axlim
[
3
],
color
=
'k'
,
linestyle
=
'--'
,
label
=
"%s @ EER"
%
self
.
_y_label
)
else
:
mpl
.
axhline
(
y
=
farfrr_licit_det
[
1
],
xmin
=
axlim
[
0
],
xmax
=
axlim
[
1
],
color
=
'k'
,
linestyle
=
'--'
,
label
=
"%s = %.2f%%"
%
(
self
.
_y_label
,
farfrr_licit
[
1
]
*
100
))
mpl
.
plot
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
],
'o'
,
color
=
'C0'
,
size
=
'small'
)
mpl
.
annotate
(
'IAPMR=%.2f%%'
%
(
farfrr_spoof
[
0
]
*
100
),
xy
=
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_spoof
[
0
],
xyannotate_spoof
[
1
]),
)
# FAR point, licit scenario
mpl
.
plot
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
],
'o'
,
color
=
'C3'
,
size
=
'small'
)
)
# FAR point, spoof scenario
# annotate the FAR points
if
farfrr_licit_det
[
0
]
<
farfrr_spoof_det
[
0
]:
xyannotate_licit
=
[
farfrr_licit_det
[
0
]
-
0.7
,
farfrr_licit_det
[
1
]
-
0.4
,
]
xyannotate_spoof
=
[
0.1
+
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]
+
0.3
,
]
else
:
xyannotate_spoof
=
[
farfrr_licit_det
[
0
]
-
0.7
,
farfrr_licit_det
[
1
]
-
0.4
,
]
xyannotate_licit
=
[
0.1
+
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]
+
0.3
,
]
if
not
self
.
_real_data
:
mpl
.
annotate
(
'%s @ operating point'
%
self
.
_y_label
,
xy
=
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_licit
[
0
],
xyannotate_licit
[
1
]))
mpl
.
annotate
(
'IAPMR @ operating point'
,
xy
=
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_spoof
[
0
],
xyannotate_spoof
[
1
]))
else
:
mpl
.
annotate
(
'FMR=%.2f%%'
%
(
farfrr_licit
[
0
]
*
100
),
xy
=
(
farfrr_licit_det
[
0
],
farfrr_licit_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_licit
[
0
],
xyannotate_licit
[
1
]),
color
=
'C0'
,
size
=
'small'
)
mpl
.
annotate
(
'IAPMR=%.2f%%'
%
(
farfrr_spoof
[
0
]
*
100
),
xy
=
(
farfrr_spoof_det
[
0
],
farfrr_spoof_det
[
1
]),
xycoords
=
'data'
,
xytext
=
(
xyannotate_spoof
[
0
],
xyannotate_spoof
[
1
]),
color
=
'C3'
,
size
=
'small'
)
def
end_process
(
self
):
''' Set title, legend, axis labels, grid colors, save figures and
...
...
@@ -619,10 +621,10 @@ class BaseVulnDetRoc(PadPlot):
class
DetVuln
(
BaseVulnDetRoc
):
'''DET for vuln'''
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
criteria
,
real_data
,
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
real_data
,
no_spoof
):
super
(
Det
,
self
).
__init__
(
ctx
,
scores
,
evaluation
,
func_load
,
criteria
,
real_data
,
no_spoof
)
real_data
,
no_spoof
)
self
.
_x_label
=
self
.
_x_label
or
"FMR"
self
.
_y_label
=
self
.
_y_label
or
"FNMR"
add
=
''
...
...
@@ -653,10 +655,9 @@ class DetVuln(BaseVulnDetRoc):
class
RocVuln
(
BaseVulnDetRoc
):
'''ROC for vuln'''
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
criteria
,
real_data
,
no_spoof
):
def
__init__
(
self
,
ctx
,
scores
,
evaluation
,
func_load
,
real_data
,
no_spoof
):
super
(
RocVuln
,
self
).
__init__
(
ctx
,
scores
,
evaluation
,
func_load
,
criteria
,
real_data
,
no_spoof
)
real_data
,
no_spoof
)
self
.
_x_label
=
self
.
_x_label
or
"FMR"
self
.
_y_label
=
self
.
_y_label
or
"1 - FNMR"
self
.
_semilogx
=
ctx
.
meta
.
get
(
'semilogx'
,
True
)
...
...
@@ -677,6 +678,7 @@ class RocVuln(BaseVulnDetRoc):
)
class
FmrIapmr
(
PadPlot
):
'''FMR vs IAPMR'''
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment