Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.measure
Commits
cba04162
Commit
cba04162
authored
Feb 07, 2017
by
Amir Mohammadi
Browse files
Remove nan from scores and report FTA instead
Report proper FMR, FNMR, FAR, FRR values
parent
1821ba71
Pipeline
#6970
passed with stages
in 21 minutes and 8 seconds
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/measure/script/compute_perf.py
View file @
cba04162
...
...
@@ -50,6 +50,7 @@ Examples:
"""
from
__future__
import
division
import
os
import
sys
import
numpy
...
...
@@ -58,7 +59,7 @@ import bob.core
logger
=
bob
.
core
.
log
.
setup
(
"bob.measure"
)
def
print_crit
(
crit
,
dev_scores
,
test_scores
=
None
):
def
print_crit
(
crit
,
dev_scores
,
dev_fta
,
test_scores
=
None
,
test_fta
=
None
):
"""Prints a single output line that contains all info for a given criterion"""
dev_neg
,
dev_pos
=
dev_scores
...
...
@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None):
thres
=
min_hter_threshold
(
dev_neg
,
dev_pos
)
from
..
import
farfrr
dev_far
,
dev_frr
=
farfrr
(
dev_neg
,
dev_pos
,
thres
)
dev_hter
=
(
dev_far
+
dev_frr
)
/
2.0
dev_fmr
,
dev_fnmr
=
farfrr
(
dev_neg
,
dev_pos
,
thres
)
dev_far
=
dev_fmr
*
(
1
-
dev_fta
)
dev_frr
=
dev_fta
+
dev_fnmr
*
(
1
-
dev_fta
)
dev_hter
=
(
dev_far
+
dev_frr
)
/
2.0
print
(
"[Min. criterion: %s] Threshold on Development set: %e"
%
(
crit
,
thres
))
dev_ni
=
dev_neg
.
shape
[
0
]
#
number of impostors
dev_f
a
=
int
(
round
(
dev_f
ar
*
dev_ni
))
#
number of false accepts
dev_nc
=
dev_pos
.
shape
[
0
]
#
number of clients
dev_f
r
=
int
(
round
(
dev_f
rr
*
dev_nc
))
#
number of false rejects
dev_ni
=
dev_neg
.
shape
[
0
]
#
number of impostors
dev_f
m
=
int
(
round
(
dev_f
mr
*
dev_ni
))
#
number of false accepts
dev_nc
=
dev_pos
.
shape
[
0
]
#
number of clients
dev_f
nm
=
int
(
round
(
dev_f
nmr
*
dev_nc
))
#
number of false rejects
dev_f
a
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
dev_f
a
r
,
dev_f
a
,
dev_ni
)
dev_f
r
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
dev_f
r
r
,
dev_f
r
,
dev_nc
)
dev_max_len
=
max
(
len
(
dev_f
a
r_str
),
len
(
dev_f
r
r_str
))
dev_f
m
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
dev_f
m
r
,
dev_f
m
,
dev_ni
)
dev_f
nm
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
dev_f
nm
r
,
dev_f
nm
,
dev_nc
)
dev_max_len
=
max
(
len
(
dev_f
m
r_str
),
len
(
dev_f
nm
r_str
))
def
fmt
(
s
,
space
):
return
(
'%'
+
(
'%d'
%
space
)
+
's'
)
%
s
...
...
@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None):
if
test_scores
is
None
:
# prints only dev performance rates
print
(
" | %s"
%
fmt
(
"Development"
,
-
1
*
dev_max_len
))
print
(
"-------+-%s"
%
(
dev_max_len
*
"-"
))
print
(
" FAR | %s"
%
fmt
(
dev_far_str
,
dev_max_len
))
print
(
" FRR | %s"
%
fmt
(
dev_frr_str
,
dev_max_len
))
dev_hter_str
=
"%.3f%%"
%
(
100
*
dev_hter
)
print
(
" HTER | %s"
%
fmt
(
dev_hter_str
,
-
1
*
dev_max_len
))
print
(
" | %s"
%
fmt
(
"Development"
,
-
1
*
dev_max_len
))
print
(
"-------+-%s"
%
(
dev_max_len
*
"-"
))
print
(
" FMR | %s"
%
fmt
(
dev_fmr_str
,
-
1
*
dev_max_len
))
print
(
" FNMR | %s"
%
fmt
(
dev_fnmr_str
,
-
1
*
dev_max_len
))
dev_far_str
=
"%.3f%%"
%
(
100
*
dev_far
)
print
(
" FAR | %s"
%
fmt
(
dev_far_str
,
-
1
*
dev_max_len
))
dev_frr_str
=
"%.3f%%"
%
(
100
*
dev_frr
)
print
(
" FRR | %s"
%
fmt
(
dev_frr_str
,
-
1
*
dev_max_len
))
dev_hter_str
=
"%.3f%%"
%
(
100
*
dev_hter
)
print
(
" HTER | %s"
%
fmt
(
dev_hter_str
,
-
1
*
dev_max_len
))
else
:
# computes statistics for the test set based on the threshold a priori
test_neg
,
test_pos
=
test_scores
test_far
,
test_frr
=
farfrr
(
test_neg
,
test_pos
,
thres
)
test_hter
=
(
test_far
+
test_frr
)
/
2.0
test_fmr
,
test_fnmr
=
farfrr
(
test_neg
,
test_pos
,
thres
)
test_far
=
test_fmr
*
(
1
-
test_fta
)
test_frr
=
test_fta
+
test_fnmr
*
(
1
-
test_fta
)
test_hter
=
(
test_far
+
test_frr
)
/
2.0
test_ni
=
test_neg
.
shape
[
0
]
#
number of impostors
test_f
a
=
int
(
round
(
test_f
ar
*
test_ni
))
#
number of false accepts
test_nc
=
test_pos
.
shape
[
0
]
#
number of clients
test_f
r
=
int
(
round
(
test_f
rr
*
test_nc
))
#
number of false rejects
test_ni
=
test_neg
.
shape
[
0
]
#
number of impostors
test_f
m
=
int
(
round
(
test_f
mr
*
test_ni
))
#
number of false accepts
test_nc
=
test_pos
.
shape
[
0
]
#
number of clients
test_f
nm
=
int
(
round
(
test_f
nmr
*
test_nc
))
#
number of false rejects
test_f
a
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
test_f
a
r
,
test_f
a
,
test_ni
)
test_f
r
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
test_f
r
r
,
test_f
r
,
test_nc
)
test_max_len
=
max
(
len
(
test_f
a
r_str
),
len
(
test_f
r
r_str
))
test_f
m
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
test_f
m
r
,
test_f
m
,
test_ni
)
test_f
nm
r_str
=
"%.3f%% (%d/%d)"
%
(
100
*
test_f
nm
r
,
test_f
nm
,
test_nc
)
test_max_len
=
max
(
len
(
test_f
m
r_str
),
len
(
test_f
nm
r_str
))
# prints both dev and test performance rates
print
(
" | %s | %s"
%
(
fmt
(
"Development"
,
-
1
*
dev_max_len
),
fmt
(
"Test"
,
-
1
*
test_max_len
)))
print
(
"-------+-%s-+-%s"
%
(
dev_max_len
*
"-"
,
(
2
+
test_max_len
)
*
"-"
))
print
(
" FAR | %s | %s"
%
(
fmt
(
dev_far_str
,
dev_max_len
),
fmt
(
test_far_str
,
test_max_len
)))
print
(
" FRR | %s | %s"
%
(
fmt
(
dev_frr_str
,
dev_max_len
),
fmt
(
test_frr_str
,
test_max_len
)))
dev_hter_str
=
"%.3f%%"
%
(
100
*
dev_hter
)
test_hter_str
=
"%.3f%%"
%
(
100
*
test_hter
)
print
(
" HTER | %s | %s"
%
(
fmt
(
dev_hter_str
,
-
1
*
dev_max_len
),
fmt
(
test_hter_str
,
-
1
*
test_max_len
)))
print
(
" | %s | %s"
%
(
fmt
(
"Development"
,
-
1
*
dev_max_len
),
fmt
(
"Test"
,
-
1
*
test_max_len
)))
print
(
"-------+-%s-+-%s"
%
(
dev_max_len
*
"-"
,
(
2
+
test_max_len
)
*
"-"
))
print
(
" FMR | %s | %s"
%
(
fmt
(
dev_fmr_str
,
-
1
*
dev_max_len
),
fmt
(
test_fmr_str
,
-
1
*
test_max_len
)))
print
(
" FNMR | %s | %s"
%
(
fmt
(
dev_fnmr_str
,
-
1
*
dev_max_len
),
fmt
(
test_fnmr_str
,
-
1
*
test_max_len
)))
dev_far_str
=
"%.3f%%"
%
(
100
*
dev_far
)
test_far_str
=
"%.3f%%"
%
(
100
*
test_far
)
print
(
" FAR | %s | %s"
%
(
fmt
(
dev_far_str
,
-
1
*
dev_max_len
),
fmt
(
test_far_str
,
-
1
*
test_max_len
)))
dev_frr_str
=
"%.3f%%"
%
(
100
*
dev_frr
)
test_frr_str
=
"%.3f%%"
%
(
100
*
test_frr
)
print
(
" FRR | %s | %s"
%
(
fmt
(
dev_frr_str
,
-
1
*
dev_max_len
),
fmt
(
test_frr_str
,
-
1
*
test_max_len
)))
dev_hter_str
=
"%.3f%%"
%
(
100
*
dev_hter
)
test_hter_str
=
"%.3f%%"
%
(
100
*
test_hter
)
print
(
" HTER | %s | %s"
%
(
fmt
(
dev_hter_str
,
-
1
*
dev_max_len
),
fmt
(
test_hter_str
,
-
1
*
test_max_len
)))
def
plots
(
crit
,
points
,
filename
,
dev_scores
,
test_scores
=
None
):
...
...
@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
from
..
import
plot
import
matplotlib
if
not
hasattr
(
matplotlib
,
'backends'
):
matplotlib
.
use
(
'pdf'
)
if
not
hasattr
(
matplotlib
,
'backends'
):
matplotlib
.
use
(
'pdf'
)
import
matplotlib.pyplot
as
mpl
from
matplotlib.backends.backend_pdf
import
PdfPages
...
...
@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig
=
mpl
.
figure
()
if
test_scores
is
not
None
:
plot
.
roc
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0.3
,
0.3
,
0.3
),
linestyle
=
'--'
,
dashes
=
(
6
,
2
),
label
=
'development'
)
plot
.
roc
(
test_neg
,
test_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'test'
)
plot
.
roc
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0.3
,
0.3
,
0.3
),
linestyle
=
'--'
,
dashes
=
(
6
,
2
),
label
=
'development'
)
plot
.
roc
(
test_neg
,
test_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'test'
)
else
:
plot
.
roc
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'development'
)
plot
.
roc
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'development'
)
mpl
.
axis
([
0
,
40
,
0
,
40
])
mpl
.
axis
([
0
,
40
,
0
,
40
])
mpl
.
title
(
"ROC Curve"
)
mpl
.
xlabel
(
'FAR (%)'
)
mpl
.
ylabel
(
'FRR (%)'
)
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
if
test_scores
is
not
None
:
mpl
.
legend
()
mpl
.
xlabel
(
'FMR (%)'
)
mpl
.
ylabel
(
'FNMR (%)'
)
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
if
test_scores
is
not
None
:
mpl
.
legend
()
pp
.
savefig
(
fig
)
# DET
fig
=
mpl
.
figure
()
if
test_scores
is
not
None
:
plot
.
det
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0.3
,
0.3
,
0.3
),
linestyle
=
'--'
,
dashes
=
(
6
,
2
),
label
=
'development'
)
plot
.
det
(
test_neg
,
test_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'test'
)
plot
.
det
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0.3
,
0.3
,
0.3
),
linestyle
=
'--'
,
dashes
=
(
6
,
2
),
label
=
'development'
)
plot
.
det
(
test_neg
,
test_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'test'
)
else
:
plot
.
det
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'development'
)
plot
.
det
(
dev_neg
,
dev_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
,
label
=
'development'
)
plot
.
det_axis
([
0.01
,
40
,
0.01
,
40
])
mpl
.
title
(
"DET Curve"
)
mpl
.
xlabel
(
'FAR (%)'
)
mpl
.
ylabel
(
'FRR (%)'
)
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
if
test_scores
is
not
None
:
mpl
.
legend
()
mpl
.
xlabel
(
'FMR (%)'
)
mpl
.
ylabel
(
'FNMR (%)'
)
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
if
test_scores
is
not
None
:
mpl
.
legend
()
pp
.
savefig
(
fig
)
# EPC - requires test set
if
test_scores
is
not
None
:
fig
=
mpl
.
figure
()
plot
.
epc
(
dev_neg
,
dev_pos
,
test_neg
,
test_pos
,
points
,
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
)
color
=
(
0
,
0
,
0
),
linestyle
=
'-'
)
mpl
.
title
(
'EPC Curve'
)
mpl
.
xlabel
(
'Cost'
)
mpl
.
ylabel
(
'Min. HTER (%)'
)
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
mpl
.
grid
(
True
,
color
=
(
0.3
,
0.3
,
0.3
))
pp
.
savefig
(
fig
)
# Distribution for dev and test scores on the same page
...
...
@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig
=
mpl
.
figure
()
if
test_scores
is
not
None
:
mpl
.
subplot
(
2
,
1
,
1
)
mpl
.
subplot
(
2
,
1
,
1
)
all_scores
=
numpy
.
hstack
((
dev_neg
,
test_neg
,
dev_pos
,
test_pos
))
else
:
all_scores
=
numpy
.
hstack
((
dev_neg
,
dev_pos
))
nbins
=
20
nbins
=
20
score_range
=
all_scores
.
min
(),
all_scores
.
max
()
mpl
.
hist
(
dev_neg
,
label
=
'Impostors'
,
normed
=
True
,
color
=
'red'
,
alpha
=
0.5
,
bins
=
nbins
)
bins
=
nbins
)
mpl
.
hist
(
dev_pos
,
label
=
'Genuine'
,
normed
=
True
,
color
=
'blue'
,
alpha
=
0.5
,
bins
=
nbins
)
bins
=
nbins
)
mpl
.
xlim
(
*
score_range
)
_
,
_
,
ymax
,
ymin
=
mpl
.
axis
()
mpl
.
vlines
(
thres
,
ymin
,
ymax
,
color
=
'black'
,
label
=
'EER'
,
linestyle
=
'dashed'
)
...
...
@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
ax
=
mpl
.
gca
()
ax
.
axes
.
get_xaxis
().
set_ticklabels
([])
mpl
.
legend
(
loc
=
'upper center'
,
ncol
=
3
,
bbox_to_anchor
=
(
0.5
,
-
0.01
),
fontsize
=
10
)
fontsize
=
10
)
mpl
.
ylabel
(
'Dev. Scores (normalized)'
)
else
:
mpl
.
ylabel
(
'Normalized Count'
)
...
...
@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
mpl
.
grid
(
True
,
alpha
=
0.5
)
if
test_scores
is
not
None
:
mpl
.
subplot
(
2
,
1
,
2
)
mpl
.
subplot
(
2
,
1
,
2
)
mpl
.
hist
(
test_neg
,
label
=
'Impostors'
,
normed
=
True
,
color
=
'red'
,
alpha
=
0.5
,
bins
=
nbins
)
bins
=
nbins
)
mpl
.
hist
(
test_pos
,
label
=
'Genuine'
,
normed
=
True
,
color
=
'blue'
,
alpha
=
0.5
,
bins
=
nbins
)
bins
=
nbins
)
mpl
.
ylabel
(
'Test Scores (normalized)'
)
mpl
.
xlabel
(
'Score value'
)
mpl
.
xlim
(
*
score_range
)
_
,
_
,
ymax
,
ymin
=
mpl
.
axis
()
mpl
.
vlines
(
thres
,
ymin
,
ymax
,
color
=
'black'
,
label
=
'EER'
,
linestyle
=
'dashed'
)
linestyle
=
'dashed'
)
mpl
.
grid
(
True
,
alpha
=
0.5
)
pp
.
savefig
(
fig
)
...
...
@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
pp
.
close
()
def
remove_nan
(
scores
):
'removes the NaNs from the scores'
nans
=
numpy
.
isnan
(
scores
)
sum_nans
=
sum
(
nans
)
total
=
len
(
scores
)
if
sum_nans
>
0
:
logger
.
warning
(
'Found {} NaNs in {} scores'
.
format
(
sum_nans
,
total
))
return
scores
[
numpy
.
where
(
~
nans
)],
sum_nans
,
total
def
get_fta
(
scores
):
'calculates the Failure To Acquire (FtA)'
fta_sum
,
fta_total
=
0
,
0
neg
,
sum_nans
,
total
=
remove_nan
(
scores
[
0
])
fta_sum
+=
sum_nans
fta_total
+=
total
pos
,
sum_nans
,
total
=
remove_nan
(
scores
[
1
])
fta_sum
+=
sum_nans
fta_total
+=
total
return
((
neg
,
pos
),
fta_sum
/
fta_total
)
def
main
(
user_input
=
None
):
if
user_input
is
not
None
:
...
...
@@ -269,13 +311,13 @@ def main(user_input=None):
completions
=
dict
(
prog
=
os
.
path
.
basename
(
sys
.
argv
[
0
]),
version
=
pkg_resources
.
require
(
'bob.measure'
)[
0
].
version
)
)
args
=
docopt
.
docopt
(
__doc__
%
completions
,
argv
=
argv
,
version
=
completions
[
'version'
],
)
)
# Sets-up logging
verbosity
=
int
(
args
[
'--verbose'
])
...
...
@@ -285,23 +327,34 @@ def main(user_input=None):
try
:
args
[
'--points'
]
=
int
(
args
[
'--points'
])
except
:
raise
docopt
.
DocoptExit
(
"cannot convert %s into int for points"
%
\
args
[
'--points'
])
raise
docopt
.
DocoptExit
(
"cannot convert %s into int for points"
%
args
[
'--points'
])
if
args
[
'--points'
]
<=
0
:
raise
docopt
.
DocoptExit
(
'Number of points (--points) should greater '
\
'than zero'
)
raise
docopt
.
DocoptExit
(
'Number of points (--points) should greater '
'than zero'
)
from
..load
import
load_score
,
get_negatives_positives
dev_scores
=
get_negatives_positives
(
load_scor
e
(
args
[
'<dev-scores>'
])
)
from
..load
import
get_negatives_positives
_from_file
dev_scores
=
get_negatives_positives
_from_fil
e
(
args
[
'<dev-scores>'
])
if
args
[
'<test-scores>'
]
is
not
None
:
test_scores
=
get_negatives_positives
(
load_scor
e
(
args
[
'<test-scores>'
])
)
test_scores
=
get_negatives_positives
_from_fil
e
(
args
[
'<test-scores>'
])
else
:
test_scores
=
None
test_fta
=
None
# test if there are nan in the score files and remove them
# also calculate FTA
dev_scores
,
dev_fta
=
get_fta
(
dev_scores
)
print
(
"Failure To Acquire (FTA) in the development set is: {:.3f}%"
.
format
(
dev_fta
*
100
))
if
test_scores
is
not
None
:
test_scores
,
test_fta
=
get_fta
(
test_scores
)
print
(
"Failure To Acquire (FTA) in the test set is: {:.3f}%"
.
format
(
test_fta
*
100
))
print_crit
(
'EER'
,
dev_scores
,
test_scores
)
print_crit
(
'Min. HTER'
,
dev_scores
,
test_scores
)
print_crit
(
'EER'
,
dev_scores
,
dev_fta
,
test_scores
,
test_fta
)
print_crit
(
'Min. HTER'
,
dev_scores
,
dev_fta
,
test_scores
,
test_fta
)
if
not
args
[
'--no-plot'
]:
plots
(
'EER'
,
args
[
'--points'
],
args
[
'--output'
],
dev_scores
,
test_scores
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment