Commit 616825ae authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'std' into 'master'

Remove nan from scores and report FTA instead

See merge request !27
parents 1821ba71 4b068ce7
Pipeline #7113 passed with stages
in 9 minutes and 52 seconds
......@@ -50,6 +50,7 @@ Examples:
"""
from __future__ import division
import os
import sys
import numpy
......@@ -58,7 +59,7 @@ import bob.core
logger = bob.core.log.setup("bob.measure")
def print_crit(crit, dev_scores, test_scores=None):
def print_crit(crit, dev_scores, dev_fta, test_scores=None, test_fta=None):
"""Prints a single output line that contains all info for a given criterion"""
dev_neg, dev_pos = dev_scores
......@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None):
thres = min_hter_threshold(dev_neg, dev_pos)
from .. import farfrr
dev_far, dev_frr = farfrr(dev_neg, dev_pos, thres)
dev_hter = (dev_far + dev_frr)/2.0
dev_fmr, dev_fnmr = farfrr(dev_neg, dev_pos, thres)
dev_far = dev_fmr * (1 - dev_fta)
dev_frr = dev_fta + dev_fnmr * (1 - dev_fta)
dev_hter = (dev_far + dev_frr) / 2.0
print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
dev_ni = dev_neg.shape[0] #number of impostors
dev_fa = int(round(dev_far*dev_ni)) #number of false accepts
dev_nc = dev_pos.shape[0] #number of clients
dev_fr = int(round(dev_frr*dev_nc)) #number of false rejects
dev_ni = dev_neg.shape[0] # number of impostors
dev_fm = int(round(dev_fmr * dev_ni)) # number of false accepts
dev_nc = dev_pos.shape[0] # number of clients
dev_fnm = int(round(dev_fnmr * dev_nc)) # number of false rejects
dev_far_str = "%.3f%% (%d/%d)" % (100*dev_far, dev_fa, dev_ni)
dev_frr_str = "%.3f%% (%d/%d)" % (100*dev_frr, dev_fr, dev_nc)
dev_max_len = max(len(dev_far_str), len(dev_frr_str))
dev_fmr_str = "%.3f%% (%d/%d)" % (100 * dev_fmr, dev_fm, dev_ni)
dev_fnmr_str = "%.3f%% (%d/%d)" % (100 * dev_fnmr, dev_fnm, dev_nc)
dev_max_len = max(len(dev_fmr_str), len(dev_fnmr_str))
def fmt(s, space):
return ('%' + ('%d' % space) + 's') % s
......@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None):
if test_scores is None:
# prints only dev performance rates
print(" | %s" % fmt("Development", -1*dev_max_len))
print("-------+-%s" % (dev_max_len*"-"))
print(" FAR | %s" % fmt(dev_far_str, dev_max_len))
print(" FRR | %s" % fmt(dev_frr_str, dev_max_len))
dev_hter_str = "%.3f%%" % (100*dev_hter)
print(" HTER | %s" % fmt(dev_hter_str, -1*dev_max_len))
print(" | %s" % fmt("Development", -1 * dev_max_len))
print("-------+-%s" % (dev_max_len * "-"))
print(" FMR | %s" % fmt(dev_fmr_str, -1 * dev_max_len))
print(" FNMR | %s" % fmt(dev_fnmr_str, -1 * dev_max_len))
dev_far_str = "%.3f%%" % (100 * dev_far)
print(" FAR | %s" % fmt(dev_far_str, -1 * dev_max_len))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
print(" FRR | %s" % fmt(dev_frr_str, -1 * dev_max_len))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
print(" HTER | %s" % fmt(dev_hter_str, -1 * dev_max_len))
else:
# computes statistics for the test set based on the threshold a priori
test_neg, test_pos = test_scores
test_far, test_frr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0
test_fmr, test_fnmr = farfrr(test_neg, test_pos, thres)
test_far = test_fmr * (1 - test_fta)
test_frr = test_fta + test_fnmr * (1 - test_fta)
test_hter = (test_far + test_frr) / 2.0
test_ni = test_neg.shape[0] #number of impostors
test_fa = int(round(test_far*test_ni)) #number of false accepts
test_nc = test_pos.shape[0] #number of clients
test_fr = int(round(test_frr*test_nc)) #number of false rejects
test_ni = test_neg.shape[0] # number of impostors
test_fm = int(round(test_fmr * test_ni)) # number of false accepts
test_nc = test_pos.shape[0] # number of clients
test_fnm = int(round(test_fnmr * test_nc)) # number of false rejects
test_far_str = "%.3f%% (%d/%d)" % (100*test_far, test_fa, test_ni)
test_frr_str = "%.3f%% (%d/%d)" % (100*test_frr, test_fr, test_nc)
test_max_len = max(len(test_far_str), len(test_frr_str))
test_fmr_str = "%.3f%% (%d/%d)" % (100 * test_fmr, test_fm, test_ni)
test_fnmr_str = "%.3f%% (%d/%d)" % (100 * test_fnmr, test_fnm, test_nc)
test_max_len = max(len(test_fmr_str), len(test_fnmr_str))
# prints both dev and test performance rates
print(" | %s | %s" % (fmt("Development", -1*dev_max_len),
fmt("Test", -1*test_max_len)))
print("-------+-%s-+-%s" % (dev_max_len*"-", (2+test_max_len)*"-"))
print(" FAR | %s | %s" % (fmt(dev_far_str, dev_max_len),
fmt(test_far_str, test_max_len)))
print(" FRR | %s | %s" % (fmt(dev_frr_str, dev_max_len),
fmt(test_frr_str, test_max_len)))
dev_hter_str = "%.3f%%" % (100*dev_hter)
test_hter_str = "%.3f%%" % (100*test_hter)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1*dev_max_len),
fmt(test_hter_str, -1*test_max_len)))
print(" | %s | %s" % (fmt("Development", -1 * dev_max_len),
fmt("Test", -1 * test_max_len)))
print("-------+-%s-+-%s" % (dev_max_len * "-", (2 + test_max_len) * "-"))
print(" FMR | %s | %s" % (fmt(dev_fmr_str, -1 * dev_max_len),
fmt(test_fmr_str, -1 * test_max_len)))
print(" FNMR | %s | %s" % (fmt(dev_fnmr_str, -1 * dev_max_len),
fmt(test_fnmr_str, -1 * test_max_len)))
dev_far_str = "%.3f%%" % (100 * dev_far)
test_far_str = "%.3f%%" % (100 * test_far)
print(" FAR | %s | %s" % (fmt(dev_far_str, -1 * dev_max_len),
fmt(test_far_str, -1 * test_max_len)))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
test_frr_str = "%.3f%%" % (100 * test_frr)
print(" FRR | %s | %s" % (fmt(dev_frr_str, -1 * dev_max_len),
fmt(test_frr_str, -1 * test_max_len)))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
test_hter_str = "%.3f%%" % (100 * test_hter)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1 * dev_max_len),
fmt(test_hter_str, -1 * test_max_len)))
def plots(crit, points, filename, dev_scores, test_scores=None):
......@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
from .. import plot
import matplotlib
if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
if not hasattr(matplotlib, 'backends'):
matplotlib.use('pdf')
import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages
......@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure()
if test_scores is not None:
plot.roc(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
plot.roc(test_neg, test_pos, points, color=(0,0,0),
linestyle='-', label='test')
plot.roc(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6, 2), label='development')
plot.roc(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test')
else:
plot.roc(dev_neg, dev_pos, points, color=(0,0,0),
linestyle='-', label='development')
plot.roc(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development')
mpl.axis([0,40,0,40])
mpl.axis([0, 40, 0, 40])
mpl.title("ROC Curve")
mpl.xlabel('FAR (%)')
mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
if test_scores is not None: mpl.legend()
mpl.xlabel('FMR (%)')
mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None:
mpl.legend()
pp.savefig(fig)
# DET
fig = mpl.figure()
if test_scores is not None:
plot.det(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
plot.det(test_neg, test_pos, points, color=(0,0,0),
linestyle='-', label='test')
plot.det(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6, 2), label='development')
plot.det(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test')
else:
plot.det(dev_neg, dev_pos, points, color=(0,0,0),
linestyle='-', label='development')
plot.det(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development')
plot.det_axis([0.01, 40, 0.01, 40])
mpl.title("DET Curve")
mpl.xlabel('FAR (%)')
mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
if test_scores is not None: mpl.legend()
mpl.xlabel('FMR (%)')
mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None:
mpl.legend()
pp.savefig(fig)
# EPC - requires test set
if test_scores is not None:
fig = mpl.figure()
plot.epc(dev_neg, dev_pos, test_neg, test_pos, points,
color=(0,0,0), linestyle='-')
color=(0, 0, 0), linestyle='-')
mpl.title('EPC Curve')
mpl.xlabel('Cost')
mpl.ylabel('Min. HTER (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.grid(True, color=(0.3, 0.3, 0.3))
pp.savefig(fig)
# Distribution for dev and test scores on the same page
......@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure()
if test_scores is not None:
mpl.subplot(2,1,1)
mpl.subplot(2, 1, 1)
all_scores = numpy.hstack((dev_neg, test_neg, dev_pos, test_pos))
else:
all_scores = numpy.hstack((dev_neg, dev_pos))
nbins=20
nbins = 20
score_range = all_scores.min(), all_scores.max()
mpl.hist(dev_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.hist(dev_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER', linestyle='dashed')
......@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
fontsize=10)
fontsize=10)
mpl.ylabel('Dev. Scores (normalized)')
else:
mpl.ylabel('Normalized Count')
......@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
mpl.grid(True, alpha=0.5)
if test_scores is not None:
mpl.subplot(2,1,2)
mpl.subplot(2, 1, 2)
mpl.hist(test_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.hist(test_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.ylabel('Test Scores (normalized)')
mpl.xlabel('Score value')
mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER',
linestyle='dashed')
linestyle='dashed')
mpl.grid(True, alpha=0.5)
pp.savefig(fig)
......@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
pp.close()
def remove_nan(scores):
"""removes the NaNs from the scores"""
nans = numpy.isnan(scores)
sum_nans = sum(nans)
total = len(scores)
if sum_nans > 0:
logger.warning('Found {} NaNs in {} scores'.format(sum_nans, total))
return scores[numpy.where(~nans)], sum_nans, total
def get_fta(scores):
"""calculates the Failure To Acquire (FtA) rate"""
fta_sum, fta_total = 0, 0
neg, sum_nans, total = remove_nan(scores[0])
fta_sum += sum_nans
fta_total += total
pos, sum_nans, total = remove_nan(scores[1])
fta_sum += sum_nans
fta_total += total
return ((neg, pos), fta_sum / fta_total)
def main(user_input=None):
if user_input is not None:
......@@ -269,13 +311,13 @@ def main(user_input=None):
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
)
# Sets-up logging
verbosity = int(args['--verbose'])
......@@ -285,23 +327,34 @@ def main(user_input=None):
try:
args['--points'] = int(args['--points'])
except:
raise docopt.DocoptExit("cannot convert %s into int for points" % \
args['--points'])
raise docopt.DocoptExit("cannot convert %s into int for points" %
args['--points'])
if args['--points'] <= 0:
raise docopt.DocoptExit('Number of points (--points) should greater ' \
'than zero')
raise docopt.DocoptExit('Number of points (--points) should greater '
'than zero')
from ..load import load_score, get_negatives_positives
dev_scores = get_negatives_positives(load_score(args['<dev-scores>']))
from ..load import get_negatives_positives_from_file
dev_scores = get_negatives_positives_from_file(args['<dev-scores>'])
if args['<test-scores>'] is not None:
test_scores = get_negatives_positives(load_score(args['<test-scores>']))
test_scores = get_negatives_positives_from_file(args['<test-scores>'])
else:
test_scores = None
test_fta = None
# test if there are nan in the score files and remove them
# also calculate FTA
dev_scores, dev_fta = get_fta(dev_scores)
print("Failure To Acquire (FTA) in the development set is: {:.3f}%".format(
dev_fta * 100))
if test_scores is not None:
test_scores, test_fta = get_fta(test_scores)
print("Failure To Acquire (FTA) in the test set is: {:.3f}%".format(
test_fta * 100))
print_crit('EER', dev_scores, test_scores)
print_crit('Min. HTER', dev_scores, test_scores)
print_crit('EER', dev_scores, dev_fta, test_scores, test_fta)
print_crit('Min. HTER', dev_scores, dev_fta, test_scores, test_fta)
if not args['--no-plot']:
plots('EER', args['--points'], args['--output'], dev_scores, test_scores)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment