Commit 616825ae authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'std' into 'master'

Remove nan from scores and report FTA instead

See merge request !27
parents 1821ba71 4b068ce7
Pipeline #7113 passed with stages
in 9 minutes and 52 seconds
......@@ -50,6 +50,7 @@ Examples:
"""
from __future__ import division
import os
import sys
import numpy
......@@ -58,7 +59,7 @@ import bob.core
logger = bob.core.log.setup("bob.measure")
def print_crit(crit, dev_scores, test_scores=None):
def print_crit(crit, dev_scores, dev_fta, test_scores=None, test_fta=None):
"""Prints a single output line that contains all info for a given criterion"""
dev_neg, dev_pos = dev_scores
......@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None):
thres = min_hter_threshold(dev_neg, dev_pos)
from .. import farfrr
dev_far, dev_frr = farfrr(dev_neg, dev_pos, thres)
dev_hter = (dev_far + dev_frr)/2.0
dev_fmr, dev_fnmr = farfrr(dev_neg, dev_pos, thres)
dev_far = dev_fmr * (1 - dev_fta)
dev_frr = dev_fta + dev_fnmr * (1 - dev_fta)
dev_hter = (dev_far + dev_frr) / 2.0
print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
dev_ni = dev_neg.shape[0] #number of impostors
dev_fa = int(round(dev_far*dev_ni)) #number of false accepts
dev_nc = dev_pos.shape[0] #number of clients
dev_fr = int(round(dev_frr*dev_nc)) #number of false rejects
dev_ni = dev_neg.shape[0] # number of impostors
dev_fm = int(round(dev_fmr * dev_ni)) # number of false accepts
dev_nc = dev_pos.shape[0] # number of clients
dev_fnm = int(round(dev_fnmr * dev_nc)) # number of false rejects
dev_far_str = "%.3f%% (%d/%d)" % (100*dev_far, dev_fa, dev_ni)
dev_frr_str = "%.3f%% (%d/%d)" % (100*dev_frr, dev_fr, dev_nc)
dev_max_len = max(len(dev_far_str), len(dev_frr_str))
dev_fmr_str = "%.3f%% (%d/%d)" % (100 * dev_fmr, dev_fm, dev_ni)
dev_fnmr_str = "%.3f%% (%d/%d)" % (100 * dev_fnmr, dev_fnm, dev_nc)
dev_max_len = max(len(dev_fmr_str), len(dev_fnmr_str))
def fmt(s, space):
return ('%' + ('%d' % space) + 's') % s
......@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None):
if test_scores is None:
# prints only dev performance rates
print(" | %s" % fmt("Development", -1*dev_max_len))
print("-------+-%s" % (dev_max_len*"-"))
print(" FAR | %s" % fmt(dev_far_str, dev_max_len))
print(" FRR | %s" % fmt(dev_frr_str, dev_max_len))
dev_hter_str = "%.3f%%" % (100*dev_hter)
print(" HTER | %s" % fmt(dev_hter_str, -1*dev_max_len))
print(" | %s" % fmt("Development", -1 * dev_max_len))
print("-------+-%s" % (dev_max_len * "-"))
print(" FMR | %s" % fmt(dev_fmr_str, -1 * dev_max_len))
print(" FNMR | %s" % fmt(dev_fnmr_str, -1 * dev_max_len))
dev_far_str = "%.3f%%" % (100 * dev_far)
print(" FAR | %s" % fmt(dev_far_str, -1 * dev_max_len))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
print(" FRR | %s" % fmt(dev_frr_str, -1 * dev_max_len))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
print(" HTER | %s" % fmt(dev_hter_str, -1 * dev_max_len))
else:
# computes statistics for the test set based on the threshold a priori
test_neg, test_pos = test_scores
test_far, test_frr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0
test_fmr, test_fnmr = farfrr(test_neg, test_pos, thres)
test_far = test_fmr * (1 - test_fta)
test_frr = test_fta + test_fnmr * (1 - test_fta)
test_hter = (test_far + test_frr) / 2.0
test_ni = test_neg.shape[0] #number of impostors
test_fa = int(round(test_far*test_ni)) #number of false accepts
test_nc = test_pos.shape[0] #number of clients
test_fr = int(round(test_frr*test_nc)) #number of false rejects
test_ni = test_neg.shape[0] # number of impostors
test_fm = int(round(test_fmr * test_ni)) # number of false accepts
test_nc = test_pos.shape[0] # number of clients
test_fnm = int(round(test_fnmr * test_nc)) # number of false rejects
test_far_str = "%.3f%% (%d/%d)" % (100*test_far, test_fa, test_ni)
test_frr_str = "%.3f%% (%d/%d)" % (100*test_frr, test_fr, test_nc)
test_max_len = max(len(test_far_str), len(test_frr_str))
test_fmr_str = "%.3f%% (%d/%d)" % (100 * test_fmr, test_fm, test_ni)
test_fnmr_str = "%.3f%% (%d/%d)" % (100 * test_fnmr, test_fnm, test_nc)
test_max_len = max(len(test_fmr_str), len(test_fnmr_str))
# prints both dev and test performance rates
print(" | %s | %s" % (fmt("Development", -1*dev_max_len),
fmt("Test", -1*test_max_len)))
print("-------+-%s-+-%s" % (dev_max_len*"-", (2+test_max_len)*"-"))
print(" FAR | %s | %s" % (fmt(dev_far_str, dev_max_len),
fmt(test_far_str, test_max_len)))
print(" FRR | %s | %s" % (fmt(dev_frr_str, dev_max_len),
fmt(test_frr_str, test_max_len)))
dev_hter_str = "%.3f%%" % (100*dev_hter)
test_hter_str = "%.3f%%" % (100*test_hter)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1*dev_max_len),
fmt(test_hter_str, -1*test_max_len)))
print(" | %s | %s" % (fmt("Development", -1 * dev_max_len),
fmt("Test", -1 * test_max_len)))
print("-------+-%s-+-%s" % (dev_max_len * "-", (2 + test_max_len) * "-"))
print(" FMR | %s | %s" % (fmt(dev_fmr_str, -1 * dev_max_len),
fmt(test_fmr_str, -1 * test_max_len)))
print(" FNMR | %s | %s" % (fmt(dev_fnmr_str, -1 * dev_max_len),
fmt(test_fnmr_str, -1 * test_max_len)))
dev_far_str = "%.3f%%" % (100 * dev_far)
test_far_str = "%.3f%%" % (100 * test_far)
print(" FAR | %s | %s" % (fmt(dev_far_str, -1 * dev_max_len),
fmt(test_far_str, -1 * test_max_len)))
dev_frr_str = "%.3f%%" % (100 * dev_frr)
test_frr_str = "%.3f%%" % (100 * test_frr)
print(" FRR | %s | %s" % (fmt(dev_frr_str, -1 * dev_max_len),
fmt(test_frr_str, -1 * test_max_len)))
dev_hter_str = "%.3f%%" % (100 * dev_hter)
test_hter_str = "%.3f%%" % (100 * test_hter)
print(" HTER | %s | %s" % (fmt(dev_hter_str, -1 * dev_max_len),
fmt(test_hter_str, -1 * test_max_len)))
def plots(crit, points, filename, dev_scores, test_scores=None):
......@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
from .. import plot
import matplotlib
if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
if not hasattr(matplotlib, 'backends'):
matplotlib.use('pdf')
import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages
......@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure()
if test_scores is not None:
plot.roc(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
plot.roc(test_neg, test_pos, points, color=(0,0,0),
linestyle='-', label='test')
plot.roc(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6, 2), label='development')
plot.roc(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test')
else:
plot.roc(dev_neg, dev_pos, points, color=(0,0,0),
linestyle='-', label='development')
plot.roc(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development')
mpl.axis([0,40,0,40])
mpl.axis([0, 40, 0, 40])
mpl.title("ROC Curve")
mpl.xlabel('FAR (%)')
mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
if test_scores is not None: mpl.legend()
mpl.xlabel('FMR (%)')
mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None:
mpl.legend()
pp.savefig(fig)
# DET
fig = mpl.figure()
if test_scores is not None:
plot.det(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
plot.det(test_neg, test_pos, points, color=(0,0,0),
linestyle='-', label='test')
plot.det(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
linestyle='--', dashes=(6, 2), label='development')
plot.det(test_neg, test_pos, points, color=(0, 0, 0),
linestyle='-', label='test')
else:
plot.det(dev_neg, dev_pos, points, color=(0,0,0),
linestyle='-', label='development')
plot.det(dev_neg, dev_pos, points, color=(0, 0, 0),
linestyle='-', label='development')
plot.det_axis([0.01, 40, 0.01, 40])
mpl.title("DET Curve")
mpl.xlabel('FAR (%)')
mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
if test_scores is not None: mpl.legend()
mpl.xlabel('FMR (%)')
mpl.ylabel('FNMR (%)')
mpl.grid(True, color=(0.3, 0.3, 0.3))
if test_scores is not None:
mpl.legend()
pp.savefig(fig)
# EPC - requires test set
if test_scores is not None:
fig = mpl.figure()
plot.epc(dev_neg, dev_pos, test_neg, test_pos, points,
color=(0,0,0), linestyle='-')
color=(0, 0, 0), linestyle='-')
mpl.title('EPC Curve')
mpl.xlabel('Cost')
mpl.ylabel('Min. HTER (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.grid(True, color=(0.3, 0.3, 0.3))
pp.savefig(fig)
# Distribution for dev and test scores on the same page
......@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
fig = mpl.figure()
if test_scores is not None:
mpl.subplot(2,1,1)
mpl.subplot(2, 1, 1)
all_scores = numpy.hstack((dev_neg, test_neg, dev_pos, test_pos))
else:
all_scores = numpy.hstack((dev_neg, dev_pos))
nbins=20
nbins = 20
score_range = all_scores.min(), all_scores.max()
mpl.hist(dev_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.hist(dev_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER', linestyle='dashed')
......@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
ax = mpl.gca()
ax.axes.get_xaxis().set_ticklabels([])
mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
fontsize=10)
fontsize=10)
mpl.ylabel('Dev. Scores (normalized)')
else:
mpl.ylabel('Normalized Count')
......@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
mpl.grid(True, alpha=0.5)
if test_scores is not None:
mpl.subplot(2,1,2)
mpl.subplot(2, 1, 2)
mpl.hist(test_neg, label='Impostors', normed=True, color='red', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.hist(test_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
bins=nbins)
bins=nbins)
mpl.ylabel('Test Scores (normalized)')
mpl.xlabel('Score value')
mpl.xlim(*score_range)
_, _, ymax, ymin = mpl.axis()
mpl.vlines(thres, ymin, ymax, color='black', label='EER',
linestyle='dashed')
linestyle='dashed')
mpl.grid(True, alpha=0.5)
pp.savefig(fig)
......@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
pp.close()
def remove_nan(scores):
"""removes the NaNs from the scores"""
nans = numpy.isnan(scores)
sum_nans = sum(nans)
total = len(scores)
if sum_nans > 0:
logger.warning('Found {} NaNs in {} scores'.format(sum_nans, total))
return scores[numpy.where(~nans)], sum_nans, total
def get_fta(scores):
"""calculates the Failure To Acquire (FtA) rate"""
fta_sum, fta_total = 0, 0
neg, sum_nans, total = remove_nan(scores[0])
fta_sum += sum_nans
fta_total += total
pos, sum_nans, total = remove_nan(scores[1])
fta_sum += sum_nans
fta_total += total
return ((neg, pos), fta_sum / fta_total)
def main(user_input=None):
if user_input is not None:
......@@ -269,13 +311,13 @@ def main(user_input=None):
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
)
# Sets-up logging
verbosity = int(args['--verbose'])
......@@ -285,23 +327,34 @@ def main(user_input=None):
try:
args['--points'] = int(args['--points'])
except:
raise docopt.DocoptExit("cannot convert %s into int for points" % \
args['--points'])
raise docopt.DocoptExit("cannot convert %s into int for points" %
args['--points'])
if args['--points'] <= 0:
raise docopt.DocoptExit('Number of points (--points) should greater ' \
'than zero')
raise docopt.DocoptExit('Number of points (--points) should greater '
'than zero')
from ..load import load_score, get_negatives_positives
dev_scores = get_negatives_positives(load_score(args['<dev-scores>']))
from ..load import get_negatives_positives_from_file
dev_scores = get_negatives_positives_from_file(args['<dev-scores>'])
if args['<test-scores>'] is not None:
test_scores = get_negatives_positives(load_score(args['<test-scores>']))
test_scores = get_negatives_positives_from_file(args['<test-scores>'])
else:
test_scores = None
test_fta = None
# test if there are nan in the score files and remove them
# also calculate FTA
dev_scores, dev_fta = get_fta(dev_scores)
print("Failure To Acquire (FTA) in the development set is: {:.3f}%".format(
dev_fta * 100))
if test_scores is not None:
test_scores, test_fta = get_fta(test_scores)
print("Failure To Acquire (FTA) in the test set is: {:.3f}%".format(
test_fta * 100))
print_crit('EER', dev_scores, test_scores)
print_crit('Min. HTER', dev_scores, test_scores)
print_crit('EER', dev_scores, dev_fta, test_scores, test_fta)
print_crit('Min. HTER', dev_scores, dev_fta, test_scores, test_fta)
if not args['--no-plot']:
plots('EER', args['--points'], args['--output'], dev_scores, test_scores)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment