Skip to content
Snippets Groups Projects
Commit 616825ae authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'std' into 'master'

Remove nan from scores and report FTA instead

See merge request !27
parents 1821ba71 4b068ce7
Branches
Tags
1 merge request!27Remove nan from scores and report FTA instead
Pipeline #
...@@ -50,6 +50,7 @@ Examples: ...@@ -50,6 +50,7 @@ Examples:
""" """
from __future__ import division
import os import os
import sys import sys
import numpy import numpy
...@@ -58,7 +59,7 @@ import bob.core ...@@ -58,7 +59,7 @@ import bob.core
logger = bob.core.log.setup("bob.measure") logger = bob.core.log.setup("bob.measure")
def print_crit(crit, dev_scores, test_scores=None): def print_crit(crit, dev_scores, dev_fta, test_scores=None, test_fta=None):
"""Prints a single output line that contains all info for a given criterion""" """Prints a single output line that contains all info for a given criterion"""
dev_neg, dev_pos = dev_scores dev_neg, dev_pos = dev_scores
...@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None): ...@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None):
thres = min_hter_threshold(dev_neg, dev_pos) thres = min_hter_threshold(dev_neg, dev_pos)
from .. import farfrr from .. import farfrr
dev_far, dev_frr = farfrr(dev_neg, dev_pos, thres) dev_fmr, dev_fnmr = farfrr(dev_neg, dev_pos, thres)
dev_hter = (dev_far + dev_frr)/2.0 dev_far = dev_fmr * (1 - dev_fta)
dev_frr = dev_fta + dev_fnmr * (1 - dev_fta)
dev_hter = (dev_far + dev_frr) / 2.0
print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres)) print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
dev_ni = dev_neg.shape[0] #number of impostors dev_ni = dev_neg.shape[0] # number of impostors
dev_fa = int(round(dev_far*dev_ni)) #number of false accepts dev_fm = int(round(dev_fmr * dev_ni)) # number of false accepts
dev_nc = dev_pos.shape[0] #number of clients dev_nc = dev_pos.shape[0] # number of clients
dev_fr = int(round(dev_frr*dev_nc)) #number of false rejects dev_fnm = int(round(dev_fnmr * dev_nc)) # number of false rejects
dev_far_str = "%.3f%% (%d/%d)" % (100*dev_far, dev_fa, dev_ni) dev_fmr_str = "%.3f%% (%d/%d)" % (100 * dev_fmr, dev_fm, dev_ni)
dev_frr_str = "%.3f%% (%d/%d)" % (100*dev_frr, dev_fr, dev_nc) dev_fnmr_str = "%.3f%% (%d/%d)" % (100 * dev_fnmr, dev_fnm, dev_nc)
dev_max_len = max(len(dev_far_str), len(dev_frr_str)) dev_max_len = max(len(dev_fmr_str), len(dev_fnmr_str))
def fmt(s, space): def fmt(s, space):
return ('%' + ('%d' % space) + 's') % s return ('%' + ('%d' % space) + 's') % s
...@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None): ...@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None):
if test_scores is None: if test_scores is None:
# prints only dev performance rates # prints only dev performance rates
print(" | %s" % fmt("Development", -1*dev_max_len)) print(" | %s" % fmt("Development", -1 * dev_max_len))
print("-------+-%s" % (dev_max_len*"-")) print("-------+-%s" % (dev_max_len * "-"))
print(" FAR | %s" % fmt(dev_far_str, dev_max_len)) print(" FMR | %s" % fmt(dev_fmr_str, -1 * dev_max_len))
print(" FRR | %s" % fmt(dev_frr_str, dev_max_len)) print(" FNMR | %s" % fmt(dev_fnmr_str, -1 * dev_max_len))
dev_hter_str = "%.3f%%" % (100*dev_hter) dev_far_str = "%.3f%%" % (100 * dev_far)
print(" HTER | %s" % fmt(dev_hter_str, -1*dev_max_len)) print(" FAR | %s" % fmt(dev_far_str, -1 * dev_max_len))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
print(" FRR | %s" % fmt(dev_frr_str, -1 * dev_max_len))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
print(" HTER | %s" % fmt(dev_hter_str, -1 * dev_max_len))
else: else:
# computes statistics for the test set based on the threshold a priori # computes statistics for the test set based on the threshold a priori
test_neg, test_pos = test_scores test_neg, test_pos = test_scores
test_far, test_frr = farfrr(test_neg, test_pos, thres) test_fmr, test_fnmr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0 test_far = test_fmr * (1 - test_fta)
test_frr = test_fta + test_fnmr * (1 - test_fta)
test_hter = (test_far + test_frr) / 2.0
test_ni = test_neg.shape[0] #number of impostors test_ni = test_neg.shape[0] # number of impostors
test_fa = int(round(test_far*test_ni)) #number of false accepts test_fm = int(round(test_fmr * test_ni)) # number of false accepts
test_nc = test_pos.shape[0] #number of clients test_nc = test_pos.shape[0] # number of clients
test_fr = int(round(test_frr*test_nc)) #number of false rejects test_fnm = int(round(test_fnmr * test_nc)) # number of false rejects
test_far_str = "%.3f%% (%d/%d)" % (100*test_far, test_fa, test_ni) test_fmr_str = "%.3f%% (%d/%d)" % (100 * test_fmr, test_fm, test_ni)
test_frr_str = "%.3f%% (%d/%d)" % (100*test_frr, test_fr, test_nc) test_fnmr_str = "%.3f%% (%d/%d)" % (100 * test_fnmr, test_fnm, test_nc)
test_max_len = max(len(test_far_str), len(test_frr_str)) test_max_len = max(len(test_fmr_str), len(test_fnmr_str))
# prints both dev and test performance rates # prints both dev and test performance rates
print(" | %s | %s" % (fmt("Development", -1*dev_max_len), print(" | %s | %s" % (fmt("Development", -1 * dev_max_len),
fmt("Test", -1*test_max_len))) fmt("Test", -1 * test_max_len)))
print("-------+-%s-+-%s" % (dev_max_len*"-", (2+test_max_len)*"-")) print("-------+-%s-+-%s" % (dev_max_len * "-", (2 + test_max_len) * "-"))
print(" FAR | %s | %s" % (fmt(dev_far_str, dev_max_len), print(" FMR | %s | %s" % (fmt(dev_fmr_str, -1 * dev_max_len),
fmt(test_far_str, test_max_len))) fmt(test_fmr_str, -1 * test_max_len)))
print(" FRR | %s | %s" % (fmt(dev_frr_str, dev_max_len), print(" FNMR | %s | %s" % (fmt(dev_fnmr_str, -1 * dev_max_len),
fmt(test_frr_str, test_max_len))) fmt(test_fnmr_str, -1 * test_max_len)))
dev_hter_str = "%.3f%%" % (100*dev_hter) dev_far_str = "%.3f%%" % (100 * dev_far)
test_hter_str = "%.3f%%" % (100*test_hter) test_far_str = "%.3f%%" % (100 * test_far)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1*dev_max_len), print(" FAR | %s | %s" % (fmt(dev_far_str, -1 * dev_max_len),
fmt(test_hter_str, -1*test_max_len))) fmt(test_far_str, -1 * test_max_len)))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
test_frr_str = "%.3f%%" % (100 * test_frr)
print(" FRR | %s | %s" % (fmt(dev_frr_str, -1 * dev_max_len),
fmt(test_frr_str, -1 * test_max_len)))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
test_hter_str = "%.3f%%" % (100 * test_hter)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1 * dev_max_len),
fmt(test_hter_str, -1 * test_max_len)))
def plots(crit, points, filename, dev_scores, test_scores=None): def plots(crit, points, filename, dev_scores, test_scores=None):
...@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
from .. import plot from .. import plot
import matplotlib import matplotlib
if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf') if not hasattr(matplotlib, 'backends'):
matplotlib.use('pdf')
import matplotlib.pyplot as mpl import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages from matplotlib.backends.backend_pdf import PdfPages
...@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure() fig = mpl.figure()
if test_scores is not None: if test_scores is not None:
plot.roc(dev_neg, dev_pos, points, color=(0.3,0.3,0.3), plot.roc(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6,2), label='development') linestyle='--', dashes=(6, 2), label='development')
plot.roc(test_neg, test_pos, points, color=(0,0,0), plot.roc(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test') linestyle='-', label='test')
else: else:
plot.roc(dev_neg, dev_pos, points, color=(0,0,0), plot.roc(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development') linestyle='-', label='development')
mpl.axis([0,40,0,40]) mpl.axis([0, 40, 0, 40])
mpl.title("ROC Curve") mpl.title("ROC Curve")
mpl.xlabel('FAR (%)') mpl.xlabel('FMR (%)')
mpl.ylabel('FRR (%)') mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3,0.3,0.3)) mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None: mpl.legend() if test_scores is not None:
mpl.legend()
pp.savefig(fig) pp.savefig(fig)
# DET # DET
fig = mpl.figure() fig = mpl.figure()
if test_scores is not None: if test_scores is not None:
plot.det(dev_neg, dev_pos, points, color=(0.3,0.3,0.3), plot.det(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6,2), label='development') linestyle='--', dashes=(6, 2), label='development')
plot.det(test_neg, test_pos, points, color=(0,0,0), plot.det(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test') linestyle='-', label='test')
else: else:
plot.det(dev_neg, dev_pos, points, color=(0,0,0), plot.det(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development') linestyle='-', label='development')
plot.det_axis([0.01, 40, 0.01, 40]) plot.det_axis([0.01, 40, 0.01, 40])
mpl.title("DET Curve") mpl.title("DET Curve")
mpl.xlabel('FAR (%)') mpl.xlabel('FMR (%)')
mpl.ylabel('FRR (%)') mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3,0.3,0.3)) mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None: mpl.legend() if test_scores is not None:
mpl.legend()
pp.savefig(fig) pp.savefig(fig)
# EPC - requires test set # EPC - requires test set
if test_scores is not None: if test_scores is not None:
fig = mpl.figure() fig = mpl.figure()
plot.epc(dev_neg, dev_pos, test_neg, test_pos, points, plot.epc(dev_neg, dev_pos, test_neg, test_pos, points,
color=(0,0,0), linestyle='-') color=(0, 0, 0), linestyle='-')
mpl.title('EPC Curve') mpl.title('EPC Curve')
mpl.xlabel('Cost') mpl.xlabel('Cost')
mpl.ylabel('Min. HTER (%)') mpl.ylabel('Min. HTER (%)')
mpl.grid(True, color=(0.3,0.3,0.3)) mpl.grid(True, color=(0.3, 0.3, 0.3))
pp.savefig(fig) pp.savefig(fig)
# Distribution for dev and test scores on the same page # Distribution for dev and test scores on the same page
...@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure() fig = mpl.figure()
if test_scores is not None: if test_scores is not None:
mpl.subplot(2,1,1) mpl.subplot(2, 1, 1)
all_scores = numpy.hstack((dev_neg, test_neg, dev_pos, test_pos)) all_scores = numpy.hstack((dev_neg, test_neg, dev_pos, test_pos))
else: else:
all_scores = numpy.hstack((dev_neg, dev_pos)) all_scores = numpy.hstack((dev_neg, dev_pos))
nbins=20 nbins = 20
score_range = all_scores.min(), all_scores.max() score_range = all_scores.min(), all_scores.max()
mpl.hist(dev_neg, label='Impostors', normed=True, color='red', alpha=0.5, mpl.hist(dev_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins) bins=nbins)
mpl.hist(dev_pos, label='Genuine', normed=True, color='blue', alpha=0.5, mpl.hist(dev_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins) bins=nbins)
mpl.xlim(*score_range) mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis() _, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER', linestyle='dashed') mpl.vlines(thres, ymin, ymax, color='black', label='EER', linestyle='dashed')
...@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
ax = mpl.gca() ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([]) ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01), mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
fontsize=10) fontsize=10)
mpl.ylabel('Dev. Scores (normalized)') mpl.ylabel('Dev. Scores (normalized)')
else: else:
mpl.ylabel('Normalized Count') mpl.ylabel('Normalized Count')
...@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
mpl.grid(True, alpha=0.5) mpl.grid(True, alpha=0.5)
if test_scores is not None: if test_scores is not None:
mpl.subplot(2,1,2) mpl.subplot(2, 1, 2)
mpl.hist(test_neg, label='Impostors', normed=True, color='red', alpha=0.5, mpl.hist(test_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins) bins=nbins)
mpl.hist(test_pos, label='Genuine', normed=True, color='blue', alpha=0.5, mpl.hist(test_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins) bins=nbins)
mpl.ylabel('Test Scores (normalized)') mpl.ylabel('Test Scores (normalized)')
mpl.xlabel('Score value') mpl.xlabel('Score value')
mpl.xlim(*score_range) mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis() _, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER', mpl.vlines(thres, ymin, ymax, color='black', label='EER',
linestyle='dashed') linestyle='dashed')
mpl.grid(True, alpha=0.5) mpl.grid(True, alpha=0.5)
pp.savefig(fig) pp.savefig(fig)
...@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None): ...@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
pp.close() pp.close()
def remove_nan(scores):
"""removes the NaNs from the scores"""
nans = numpy.isnan(scores)
sum_nans = sum(nans)
total = len(scores)
if sum_nans > 0:
logger.warning('Found {} NaNs in {} scores'.format(sum_nans, total))
return scores[numpy.where(~nans)], sum_nans, total
def get_fta(scores):
"""calculates the Failure To Acquire (FtA) rate"""
fta_sum, fta_total = 0, 0
neg, sum_nans, total = remove_nan(scores[0])
fta_sum += sum_nans
fta_total += total
pos, sum_nans, total = remove_nan(scores[1])
fta_sum += sum_nans
fta_total += total
return ((neg, pos), fta_sum / fta_total)
def main(user_input=None): def main(user_input=None):
if user_input is not None: if user_input is not None:
...@@ -269,13 +311,13 @@ def main(user_input=None): ...@@ -269,13 +311,13 @@ def main(user_input=None):
completions = dict( completions = dict(
prog=os.path.basename(sys.argv[0]), prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version version=pkg_resources.require('bob.measure')[0].version
) )
args = docopt.docopt( args = docopt.docopt(
__doc__ % completions, __doc__ % completions,
argv=argv, argv=argv,
version=completions['version'], version=completions['version'],
) )
# Sets-up logging # Sets-up logging
verbosity = int(args['--verbose']) verbosity = int(args['--verbose'])
...@@ -285,23 +327,34 @@ def main(user_input=None): ...@@ -285,23 +327,34 @@ def main(user_input=None):
try: try:
args['--points'] = int(args['--points']) args['--points'] = int(args['--points'])
except: except:
raise docopt.DocoptExit("cannot convert %s into int for points" % \ raise docopt.DocoptExit("cannot convert %s into int for points" %
args['--points']) args['--points'])
if args['--points'] <= 0: if args['--points'] <= 0:
raise docopt.DocoptExit('Number of points (--points) should greater ' \ raise docopt.DocoptExit('Number of points (--points) should greater '
'than zero') 'than zero')
from ..load import load_score, get_negatives_positives from ..load import get_negatives_positives_from_file
dev_scores = get_negatives_positives(load_score(args['<dev-scores>'])) dev_scores = get_negatives_positives_from_file(args['<dev-scores>'])
if args['<test-scores>'] is not None: if args['<test-scores>'] is not None:
test_scores = get_negatives_positives(load_score(args['<test-scores>'])) test_scores = get_negatives_positives_from_file(args['<test-scores>'])
else: else:
test_scores = None test_scores = None
test_fta = None
# test if there are nan in the score files and remove them
# also calculate FTA
dev_scores, dev_fta = get_fta(dev_scores)
print("Failure To Acquire (FTA) in the development set is: {:.3f}%".format(
dev_fta * 100))
if test_scores is not None:
test_scores, test_fta = get_fta(test_scores)
print("Failure To Acquire (FTA) in the test set is: {:.3f}%".format(
test_fta * 100))
print_crit('EER', dev_scores, test_scores) print_crit('EER', dev_scores, dev_fta, test_scores, test_fta)
print_crit('Min. HTER', dev_scores, test_scores) print_crit('Min. HTER', dev_scores, dev_fta, test_scores, test_fta)
if not args['--no-plot']: if not args['--no-plot']:
plots('EER', args['--points'], args['--output'], dev_scores, test_scores) plots('EER', args['--points'], args['--output'], dev_scores, test_scores)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment