Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pad.base
Commits
1358cfab
Commit
1358cfab
authored
Apr 23, 2018
by
Theophile GENTILHOMME
Browse files
Modify code according to the changes in bob.measure
parent
e477cb6f
Pipeline
#19326
failed with stage
in 62 minutes and 17 seconds
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/pad/base/script/det.py
View file @
1358cfab
...
...
@@ -6,10 +6,8 @@ from bob.extension.scripts.click_helper import verbosity_option
from
bob.bio.base.score
import
load
from
.
import
figure
FUNC_SPLIT
=
lambda
x
:
load
.
load_files
(
x
,
load
.
split
)
@
click
.
command
()
@
common_options
.
scores_argument
(
eval_mandatory
=
True
,
min_len
=
2
,
nargs
=-
1
)
@
common_options
.
scores_argument
(
min_arg
=
2
,
force_eval
=
True
,
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'det.pdf'
)
@
common_options
.
titles_option
()
@
common_options
.
const_layout_option
()
...
...
@@ -48,5 +46,5 @@ def det(ctx, scores, criteria, real_data, **kwargs):
$ bob pad det {licit,spoof}/scores-{dev,eval}
"""
process
=
figure
.
Det
(
ctx
,
scores
,
True
,
FUNC_SPLIT
,
criteria
,
real_data
)
process
=
figure
.
Det
(
ctx
,
scores
,
True
,
load
.
split
,
criteria
,
real_data
)
process
.
run
()
bob/pad/base/script/epc.py
View file @
1358cfab
...
...
@@ -6,10 +6,8 @@ from bob.extension.scripts.click_helper import verbosity_option
from
bob.bio.base.score
import
load
from
.
import
figure
FUNC_SPLIT
=
lambda
x
:
load
.
load_files
(
x
,
load
.
split
)
@
click
.
command
()
@
common_options
.
scores_argument
(
eval_mandatory
=
True
,
min_len
=
2
,
nargs
=-
1
)
@
common_options
.
scores_argument
(
min_arg
=
2
,
force_eval
=
True
,
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'epc.pdf'
)
@
common_options
.
titles_option
()
@
common_options
.
const_layout_option
()
...
...
@@ -43,11 +41,11 @@ def epc(ctx, scores, **kwargs):
$ bob pad epc {licit,spoof}/scores-{dev,eval}
"""
process
=
figure
.
Epc
(
ctx
,
scores
,
True
,
FUNC_SPLIT
)
process
=
figure
.
Epc
(
ctx
,
scores
,
True
,
load
.
split
)
process
.
run
()
@
click
.
command
()
@
common_options
.
scores_argument
(
eval_mandatory
=
True
,
min_len
=
2
,
nargs
=-
1
)
@
common_options
.
scores_argument
(
min_arg
=
2
,
force_eval
=
True
,
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'epsc.pdf'
)
@
common_options
.
titles_option
()
@
common_options
.
const_layout_option
()
...
...
@@ -102,12 +100,12 @@ def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, **kwargs):
if
(
ctx
.
meta
[
'wer'
]
and
ctx
.
meta
[
'iapmr'
]):
raise
click
.
BadParameter
(
'Cannot plot both WER and IAPMR in 3D'
)
process
=
figure
.
Epsc3D
(
ctx
,
scores
,
True
,
FUNC_SPLIT
,
ctx
,
scores
,
True
,
load
.
split
,
criteria
,
var_param
,
fixed_param
)
else
:
process
=
figure
.
Epsc
(
ctx
,
scores
,
True
,
FUNC_SPLIT
,
ctx
,
scores
,
True
,
load
.
split
,
criteria
,
var_param
,
fixed_param
)
process
.
run
()
bob/pad/base/script/figure.py
View file @
1358cfab
...
...
@@ -8,7 +8,7 @@ import matplotlib.pyplot as mpl
import
bob.measure.script.figure
as
measure_figure
from
tabulate
import
tabulate
from
bob.extension.scripts.click_helper
import
verbosity_option
from
bob.measure.utils
import
(
get_fta
,
get_thres
)
from
bob.measure.utils
import
(
get_fta
,
get_fta_list
,
get_thres
)
from
bob.measure
import
(
far_threshold
,
eer_threshold
,
min_hter_threshold
,
farfrr
,
epc
,
ppndf
)
...
...
@@ -57,15 +57,18 @@ class Metrics(measure_figure.Metrics):
super
(
Metrics
,
self
).
__init__
(
ctx
,
scores
,
evaluation
,
func_load
)
''' Compute metrics from score files'''
def
compute
(
self
,
idx
,
dev_score
,
dev_file
=
None
,
eval_score
=
None
,
eval_file
=
None
):
def
compute
(
self
,
idx
,
input_scores
,
input_names
):
''' Compute metrics for the given criteria'''
dev_neg
,
dev_pos
,
_
,
eval_neg
,
eval_pos
,
_
=
\
self
.
_process_scores
(
dev_score
,
eval_score
)
neg_list
,
pos_list
,
_
=
get_fta_list
(
input_scores
)
dev_neg
,
dev_pos
=
neg_list
[
0
],
pos_list
[
0
]
dev_file
=
input_names
[
0
]
if
self
.
_eval
:
eval_neg
,
eval_pos
=
neg_list
[
1
],
pos_list
[
1
]
eval_file
=
input_names
[
1
]
title
=
self
.
_titles
[
idx
]
if
self
.
_titles
is
not
None
else
None
headers
=
[
''
or
title
,
'Development %s'
%
dev_file
]
if
self
.
_eval
and
eval_score
is
not
None
:
if
self
.
_eval
:
headers
.
append
(
'Eval. % s'
%
eval_file
)
for
m
in
ALL_CRITERIA
:
raws
=
[]
...
...
@@ -93,11 +96,11 @@ class HistPad(measure_figure.Hist):
def
_setup_hist
(
self
,
neg
,
pos
):
self
.
_title_base
=
'PAD'
self
.
_density_hist
(
pos
,
label
=
'Bona Fide'
,
color
=
'C1'
,
**
self
.
_kwargs
pos
[
0
]
,
label
=
'Bona Fide'
,
color
=
'C1'
)
self
.
_density_hist
(
neg
,
label
=
'Presentation attack'
,
alpha
=
0.4
,
color
=
'C7'
,
hatch
=
'
\\\\
'
,
**
self
.
_kwargs
neg
[
0
]
,
label
=
'Presentation attack'
,
alpha
=
0.4
,
color
=
'C7'
,
hatch
=
'
\\\\
'
)
def
_calc_pass_rate
(
threshold
,
scores
):
...
...
@@ -134,48 +137,24 @@ def _iapmr_line_plot(scores, n_points=100, **kwargs):
mpl
.
plot
(
thres
,
mix_prob_y
,
label
=
'IAPMR'
,
color
=
'C3'
,
**
kwargs
)
def
_iapmr_plot
(
scores
,
threshold
,
iapmr
,
real_data
,
**
kwargs
):
_iapmr_dot
(
threshold
,
iapmr
,
real_data
,
**
kwargs
)
_iapmr_line_plot
(
scores
,
n_points
=
100
,
**
kwargs
)
class
HistVuln
(
measure_figure
.
Hist
):
''' Histograms for vulnerability '''
def
_get_neg_pos_thres
(
self
,
idx
,
dev_score
,
eval_score
):
assert
len
(
dev_score
)
==
self
.
_min_arg
dev_neg_list
=
[]
eval_neg_list
=
[]
dev_pos_list
=
[]
eval_pos_list
=
[]
for
i
in
range
(
self
.
_min_arg
):
dev_neg
,
dev_pos
,
_
,
eval_neg
,
eval_pos
,
_
=
self
.
_process_scores
(
dev_score
[
i
],
eval_score
[
i
]
)
dev_neg_list
.
append
(
dev_neg
)
dev_pos_list
.
append
(
dev_pos
)
eval_neg_list
.
append
(
eval_neg
)
eval_pos_list
.
append
(
eval_pos
)
threshold
=
get_thres
(
self
.
_criter
,
dev_neg_list
[
0
],
dev_pos_list
[
0
]
)
if
self
.
_thres
is
None
else
self
.
_thres
[
idx
]
return
(
dev_neg_list
,
dev_pos_list
,
eval_neg_list
,
eval_pos_list
,
threshold
)
def
_setup_hist
(
self
,
neg
,
pos
):
self
.
_title_base
=
'Vulnerability'
assert
len
(
neg
)
==
len
(
pos
)
==
self
.
_min_arg
self
.
_density_hist
(
pos
[
0
],
label
=
'Bona Fide'
,
color
=
'C1'
,
**
self
.
_kwargs
pos
[
0
],
label
=
'Bona Fide'
,
color
=
'C1'
)
self
.
_density_hist
(
neg
[
0
],
label
=
'Zero-effort impostors'
,
alpha
=
0.8
,
color
=
'C0'
,
**
self
.
_kwargs
neg
[
0
],
label
=
'Zero-effort impostors'
,
alpha
=
0.8
,
color
=
'C0'
)
self
.
_density_hist
(
neg
[
1
],
label
=
'Presentation attack'
,
alpha
=
0.4
,
color
=
'C7'
,
hatch
=
'
\\\\
'
,
**
self
.
_kwargs
hatch
=
'
\\\\
'
)
def
_lines
(
self
,
threshold
,
neg
,
pos
,
**
kwargs
):
...
...
@@ -210,24 +189,6 @@ class PadPlot(measure_figure.PlotBase):
super
(
PadPlot
,
self
).
__init__
(
ctx
,
scores
,
evaluation
,
func_load
)
mpl
.
rcParams
[
'figure.constrained_layout.use'
]
=
self
.
_clayout
def
_process_scores
(
self
,
dev_score
,
eval_score
):
'''Process score files and return neg/pos/fta for eval and dev'''
assert
len
(
dev_score
)
==
self
.
_min_arg
dev_neg_list
=
[]
eval_neg_list
=
[]
dev_pos_list
=
[]
eval_pos_list
=
[]
for
i
in
range
(
self
.
_min_arg
):
dev_neg
,
dev_pos
,
_
,
eval_neg
,
eval_pos
,
_
=
\
super
(
PadPlot
,
self
).
_process_scores
(
dev_score
[
i
],
eval_score
[
i
]
)
dev_neg_list
.
append
(
dev_neg
)
dev_pos_list
.
append
(
dev_pos
)
eval_neg_list
.
append
(
eval_neg
)
eval_pos_list
.
append
(
eval_pos
)
return
(
dev_neg_list
,
dev_pos_list
,
None
,
eval_neg_list
,
eval_pos_list
,
None
)
def
end_process
(
self
):
'''Close pdf '''
...
...
@@ -260,14 +221,19 @@ class Epc(PadPlot):
self
.
_eval
=
True
#always eval data with EPC
self
.
_split
=
False
self
.
_nb_figs
=
1
self
.
_title
=
''
def
compute
(
self
,
idx
,
dev_score
,
dev_file
,
eval_score
,
eval_file
=
None
):
if
self
.
_min_arg
!=
4
:
raise
click
.
BadParameter
(
"You must provide 4 scores files:{licit,"
"spoof}/{dev,eval}"
)
def
compute
(
self
,
idx
,
input_scores
,
input_names
):
''' Plot EPC for PAD'''
licit_dev_neg
=
dev
_score
[
0
][
0
]
licit_dev_pos
=
dev
_score
[
0
][
1
]
licit_eval_neg
=
eval
_score
[
0
][
0
]
licit_eval_pos
=
eval
_score
[
0
][
1
]
spoof_eval_neg
=
eval
_score
[
1
][
0
]
licit_dev_neg
=
input
_score
s
[
0
][
0
]
licit_dev_pos
=
input
_score
s
[
0
][
1
]
licit_eval_neg
=
input
_score
s
[
1
][
0
]
licit_eval_pos
=
input
_score
s
[
1
][
1
]
spoof_eval_neg
=
input
_score
s
[
3
][
0
]
mpl
.
gcf
().
clear
()
epc_baseline
=
epc
(
licit_dev_neg
,
licit_dev_pos
,
licit_eval_neg
,
...
...
@@ -277,7 +243,7 @@ class Epc(PadPlot):
epc_baseline
[:,
0
],
[
100.
*
k
for
k
in
epc_baseline
[:,
1
]],
color
=
'C0'
,
label
=
self
.
_label
(
'WER'
,
'%s-%s'
%
(
dev_file
[
0
],
eval_file
[
0
]),
idx
'WER'
,
'%s-%s'
%
(
input_names
[
0
],
input_names
[
1
]),
idx
),
linestyle
=
'-'
)
...
...
@@ -299,7 +265,7 @@ class Epc(PadPlot):
color
=
'C3'
,
linestyle
=
'-'
,
label
=
self
.
_label
(
'IAPMR'
,
'%s-%s'
%
(
dev_file
[
0
],
eval_file
[
0
]),
idx
'IAPMR'
,
'%s-%s'
%
(
input_names
[
0
],
input_names
[
1
]),
idx
)
)
prob_ax
.
set_yticklabels
(
prob_ax
.
get_yticks
())
...
...
@@ -330,21 +296,25 @@ class Epsc(PadPlot):
self
.
_criteria
=
'eer'
if
criteria
is
None
else
criteria
self
.
_var_param
=
"omega"
if
var_param
is
None
else
var_param
self
.
_fixed_param
=
0.5
if
fixed_param
is
None
else
fixed_param
self
.
_title
=
''
self
.
_eval
=
True
#always eval data with EPC
self
.
_split
=
False
self
.
_nb_figs
=
1
self
.
_title
=
''
if
self
.
_min_arg
!=
4
:
raise
click
.
BadParameter
(
"You must provide 4 scores files:{licit,"
"spoof}/{dev,eval}"
)
def
compute
(
self
,
idx
,
dev_score
,
dev_file
,
eval
_score
,
eval_file
=
None
):
def
compute
(
self
,
idx
,
input
_score
s
,
input_names
):
''' Plot EPSC for PAD'''
licit_dev_neg
=
dev
_score
[
0
][
0
]
licit_dev_pos
=
dev
_score
[
0
][
1
]
licit_eval_neg
=
eval
_score
[
0
][
0
]
licit_eval_pos
=
eval
_score
[
0
][
1
]
spoof_dev_neg
=
dev
_score
[
1
][
0
]
spoof_dev_pos
=
dev
_score
[
1
][
1
]
spoof_eval_neg
=
eval
_score
[
1
][
0
]
spoof_eval_pos
=
eval
_score
[
1
][
1
]
licit_dev_neg
=
input
_score
s
[
0
][
0
]
licit_dev_pos
=
input
_score
s
[
0
][
1
]
licit_eval_neg
=
input
_score
s
[
1
][
0
]
licit_eval_pos
=
input
_score
s
[
1
][
1
]
spoof_dev_neg
=
input
_score
s
[
2
][
0
]
spoof_dev_pos
=
input
_score
s
[
2
][
1
]
spoof_eval_neg
=
input
_score
s
[
3
][
0
]
spoof_eval_pos
=
input
_score
s
[
3
][
1
]
title
=
self
.
_titles
[
idx
]
if
self
.
_titles
is
not
None
else
None
mpl
.
gcf
().
clear
()
...
...
@@ -443,16 +413,16 @@ class Epsc(PadPlot):
class
Epsc3D
(
Epsc
):
''' 3D EPSC plots for PAD'''
def
compute
(
self
,
idx
,
dev_score
,
dev_file
,
eval
_score
,
eval_file
=
None
):
def
compute
(
self
,
idx
,
input
_score
s
,
input_names
):
''' Implements plots'''
licit_dev_neg
=
dev
_score
[
0
][
0
]
licit_dev_pos
=
dev
_score
[
0
][
1
]
licit_eval_neg
=
eval
_score
[
0
][
0
]
licit_eval_pos
=
eval
_score
[
0
][
1
]
spoof_dev_neg
=
dev
_score
[
1
][
0
]
spoof_dev_pos
=
dev
_score
[
1
][
1
]
spoof_eval_neg
=
eval
_score
[
1
][
0
]
spoof_eval_pos
=
eval
_score
[
1
][
1
]
licit_dev_neg
=
input
_score
s
[
0
][
0
]
licit_dev_pos
=
input
_score
s
[
0
][
1
]
licit_eval_neg
=
input
_score
s
[
1
][
0
]
licit_eval_pos
=
input
_score
s
[
1
][
1
]
spoof_dev_neg
=
input
_score
s
[
2
][
0
]
spoof_dev_pos
=
input
_score
s
[
2
][
1
]
spoof_eval_neg
=
input
_score
s
[
3
][
0
]
spoof_eval_pos
=
input
_score
s
[
3
][
1
]
title
=
self
.
_titles
[
idx
]
if
self
.
_titles
is
not
None
else
None
...
...
@@ -516,14 +486,14 @@ class Det(PadPlot):
self
.
_criteria
=
criteria
self
.
_real_data
=
True
if
real_data
is
None
else
real_data
def
compute
(
self
,
idx
,
dev_score
,
dev_file
,
eval
_score
,
eval_file
=
None
):
def
compute
(
self
,
idx
,
input
_score
s
,
input_names
):
''' Implements plots'''
licit_dev_neg
=
dev
_score
[
0
][
0
]
licit_dev_pos
=
dev
_score
[
0
][
1
]
licit_eval_neg
=
eval
_score
[
0
][
0
]
licit_eval_pos
=
eval
_score
[
0
][
1
]
spoof_eval_neg
=
eval
_score
[
1
][
0
]
if
len
(
dev
_score
)
>
1
else
None
spoof_eval_pos
=
eval
_score
[
1
][
1
]
if
len
(
dev
_score
)
>
1
else
None
licit_dev_neg
=
input
_score
s
[
0
][
0
]
licit_dev_pos
=
input
_score
s
[
0
][
1
]
licit_eval_neg
=
input
_score
s
[
1
][
0
]
licit_eval_pos
=
input
_score
s
[
1
][
1
]
spoof_eval_neg
=
input
_score
s
[
3
][
0
]
if
len
(
input
_score
s
)
>
2
else
None
spoof_eval_pos
=
input
_score
s
[
3
][
1
]
if
len
(
input
_score
s
)
>
2
else
None
det
(
licit_eval_neg
,
...
...
@@ -531,7 +501,7 @@ class Det(PadPlot):
self
.
_points
,
color
=
self
.
_colors
[
idx
],
linestyle
=
'-'
,
label
=
self
.
_label
(
"licit"
,
dev_file
[
0
],
idx
)
label
=
self
.
_label
(
"licit"
,
input_names
[
0
],
idx
)
)
if
not
self
.
_no_spoof
and
spoof_eval_neg
is
not
None
:
det
(
...
...
@@ -540,7 +510,7 @@ class Det(PadPlot):
self
.
_points
,
color
=
self
.
_colors
[
idx
],
linestyle
=
'--'
,
label
=
self
.
_label
(
"spoof"
,
eval_file
[
0
],
idx
)
label
=
self
.
_label
(
"spoof"
,
input_names
[
3
],
idx
)
)
if
self
.
_criteria
is
None
:
...
...
bob/pad/base/script/histograms.py
View file @
1358cfab
...
...
@@ -6,8 +6,6 @@ from bob.extension.scripts.click_helper import (verbosity_option, bool_option)
from
bob.bio.base.score
import
load
from
.
import
figure
FUNC_SPLIT
=
lambda
x
:
load
.
load_files
(
x
,
load
.
split
)
@
click
.
command
()
@
common_options
.
scores_argument
(
nargs
=-
1
)
@
common_options
.
output_plot_file_option
(
default_out
=
'hist.pdf'
)
...
...
@@ -44,11 +42,11 @@ def hist(ctx, scores, evaluation, **kwargs):
$ bob pad hist --criter hter dev-scores1 eval-scores1
"""
process
=
figure
.
HistPad
(
ctx
,
scores
,
evaluation
,
FUNC_SPLIT
)
process
=
figure
.
HistPad
(
ctx
,
scores
,
evaluation
,
load
.
split
)
process
.
run
()
@
click
.
command
()
@
common_options
.
scores_argument
(
nargs
=-
1
,
eval_mandatory
=
True
,
min_len
=
2
)
@
common_options
.
scores_argument
(
nargs
=-
1
,
min_arg
=
2
)
@
common_options
.
output_plot_file_option
(
default_out
=
'vuln.pdf'
)
@
common_options
.
eval_option
()
@
common_options
.
n_bins_option
()
...
...
@@ -102,5 +100,5 @@ def vuln(ctx, scores, evaluation, **kwargs):
$ bob pad vuln {licit,spoof}/scores-{dev,eval}
'''
process
=
figure
.
HistVuln
(
ctx
,
scores
,
evaluation
,
FUNC_SPLIT
)
process
=
figure
.
HistVuln
(
ctx
,
scores
,
evaluation
,
load
.
split
)
process
.
run
()
bob/pad/base/script/metrics.py
View file @
1358cfab
...
...
@@ -7,8 +7,6 @@ from bob.extension.scripts.click_helper import (verbosity_option,
from
bob.bio.base.score
import
load
from
.
import
figure
FUNC_SPLIT
=
lambda
x
:
load
.
load_files
(
x
,
load
.
split
)
@
click
.
command
(
context_settings
=
dict
(
token_normalize_func
=
lambda
x
:
x
.
lower
()))
@
common_options
.
scores_argument
(
nargs
=-
1
)
@
common_options
.
eval_option
()
...
...
@@ -51,5 +49,5 @@ def metrics(ctx, scores, evaluation, **kwargs):
$ bob pad metrics /path/to/system{1,2,3}/score-{dev,eval}
"""
process
=
figure
.
Metrics
(
ctx
,
scores
,
evaluation
,
FUNC_SPLIT
)
process
=
figure
.
Metrics
(
ctx
,
scores
,
evaluation
,
load
.
split
)
process
.
run
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment