Remove nan from scores and report FTA instead

Report proper FMR, FNMR, FAR, FRR values

bob/measure/script/compute_perf.py

@@ -50,6 +50,7 @@ Examples:
 
 """
 
+from __future__ import division
 import os
 import sys
 import numpy
@@ -58,7 +59,7 @@ import bob.core
 logger = bob.core.log.setup("bob.measure")
 
 
-def print_crit(crit, dev_scores, test_scores=None):
+def print_crit(crit, dev_scores, dev_fta, test_scores=None, test_fta=None):
   """Prints a single output line that contains all info for a given criterion"""
 
   dev_neg, dev_pos = dev_scores
@@ -71,19 +72,21 @@ def print_crit(crit, dev_scores, test_scores=None):
     thres = min_hter_threshold(dev_neg, dev_pos)
 
   from .. import farfrr
-  dev_far, dev_frr = farfrr(dev_neg, dev_pos, thres)
-  dev_hter = (dev_far + dev_frr)/2.0
+  dev_fmr, dev_fnmr = farfrr(dev_neg, dev_pos, thres)
+  dev_far = dev_fmr * (1 - dev_fta)
+  dev_frr = dev_fta + dev_fnmr * (1 - dev_fta)
+  dev_hter = (dev_far + dev_frr) / 2.0
 
   print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
 
-  dev_ni = dev_neg.shape[0] #number of impostors
-  dev_fa = int(round(dev_far*dev_ni)) #number of false accepts
-  dev_nc = dev_pos.shape[0] #number of clients
-  dev_fr = int(round(dev_frr*dev_nc)) #number of false rejects
+  dev_ni = dev_neg.shape[0]  # number of impostors
+  dev_fm = int(round(dev_fmr * dev_ni))  # number of false accepts
+  dev_nc = dev_pos.shape[0]  # number of clients
+  dev_fnm = int(round(dev_fnmr * dev_nc))  # number of false rejects
 
-  dev_far_str = "%.3f%% (%d/%d)" % (100*dev_far, dev_fa, dev_ni)
-  dev_frr_str = "%.3f%% (%d/%d)" % (100*dev_frr, dev_fr, dev_nc)
-  dev_max_len = max(len(dev_far_str), len(dev_frr_str))
+  dev_fmr_str = "%.3f%% (%d/%d)" % (100 * dev_fmr, dev_fm, dev_ni)
+  dev_fnmr_str = "%.3f%% (%d/%d)" % (100 * dev_fnmr, dev_fnm, dev_nc)
+  dev_max_len = max(len(dev_fmr_str), len(dev_fnmr_str))
 
   def fmt(s, space):
     return ('%' + ('%d' % space) + 's') % s
@@ -91,42 +94,56 @@ def print_crit(crit, dev_scores, test_scores=None):
   if test_scores is None:
 
     # prints only dev performance rates
-    print("       | %s" % fmt("Development", -1*dev_max_len))
-    print("-------+-%s" % (dev_max_len*"-"))
-    print("  FAR  | %s" % fmt(dev_far_str, dev_max_len))
-    print("  FRR  | %s" % fmt(dev_frr_str, dev_max_len))
-    dev_hter_str = "%.3f%%" % (100*dev_hter)
-    print("  HTER | %s" % fmt(dev_hter_str, -1*dev_max_len))
+    print("       | %s" % fmt("Development", -1 * dev_max_len))
+    print("-------+-%s" % (dev_max_len * "-"))
+    print("  FMR  | %s" % fmt(dev_fmr_str, -1 * dev_max_len))
+    print("  FNMR | %s" % fmt(dev_fnmr_str, -1 * dev_max_len))
+    dev_far_str = "%.3f%%" % (100 * dev_far)
+    print("  FAR  | %s" % fmt(dev_far_str, -1 * dev_max_len))
+    dev_frr_str = "%.3f%%" % (100 * dev_frr)
+    print("  FRR  | %s" % fmt(dev_frr_str, -1 * dev_max_len))
+    dev_hter_str = "%.3f%%" % (100 * dev_hter)
+    print("  HTER | %s" % fmt(dev_hter_str, -1 * dev_max_len))
 
   else:
 
     # computes statistics for the test set based on the threshold a priori
     test_neg, test_pos = test_scores
 
-    test_far, test_frr = farfrr(test_neg, test_pos, thres)
-    test_hter = (test_far + test_frr)/2.0
+    test_fmr, test_fnmr = farfrr(test_neg, test_pos, thres)
+    test_far = test_fmr * (1 - test_fta)
+    test_frr = test_fta + test_fnmr * (1 - test_fta)
+    test_hter = (test_far + test_frr) / 2.0
 
-    test_ni = test_neg.shape[0] #number of impostors
-    test_fa = int(round(test_far*test_ni)) #number of false accepts
-    test_nc = test_pos.shape[0] #number of clients
-    test_fr = int(round(test_frr*test_nc)) #number of false rejects
+    test_ni = test_neg.shape[0]  # number of impostors
+    test_fm = int(round(test_fmr * test_ni))  # number of false accepts
+    test_nc = test_pos.shape[0]  # number of clients
+    test_fnm = int(round(test_fnmr * test_nc))  # number of false rejects
 
-    test_far_str = "%.3f%% (%d/%d)" % (100*test_far, test_fa, test_ni)
-    test_frr_str = "%.3f%% (%d/%d)" % (100*test_frr, test_fr, test_nc)
-    test_max_len = max(len(test_far_str), len(test_frr_str))
+    test_fmr_str = "%.3f%% (%d/%d)" % (100 * test_fmr, test_fm, test_ni)
+    test_fnmr_str = "%.3f%% (%d/%d)" % (100 * test_fnmr, test_fnm, test_nc)
+    test_max_len = max(len(test_fmr_str), len(test_fnmr_str))
 
     # prints both dev and test performance rates
-    print("       | %s | %s" % (fmt("Development", -1*dev_max_len),
-      fmt("Test", -1*test_max_len)))
-    print("-------+-%s-+-%s" % (dev_max_len*"-", (2+test_max_len)*"-"))
-    print("  FAR  | %s | %s" % (fmt(dev_far_str, dev_max_len),
-      fmt(test_far_str, test_max_len)))
-    print("  FRR  | %s | %s" % (fmt(dev_frr_str, dev_max_len),
-      fmt(test_frr_str, test_max_len)))
-    dev_hter_str = "%.3f%%" % (100*dev_hter)
-    test_hter_str = "%.3f%%" % (100*test_hter)
-    print("  HTER | %s | %s" % (fmt(dev_hter_str, -1*dev_max_len),
-      fmt(test_hter_str, -1*test_max_len)))
+    print("       | %s | %s" % (fmt("Development", -1 * dev_max_len),
+                                fmt("Test", -1 * test_max_len)))
+    print("-------+-%s-+-%s" % (dev_max_len * "-", (2 + test_max_len) * "-"))
+    print("  FMR  | %s | %s" % (fmt(dev_fmr_str, -1 * dev_max_len),
+                                fmt(test_fmr_str, -1 * test_max_len)))
+    print("  FNMR | %s | %s" % (fmt(dev_fnmr_str, -1 * dev_max_len),
+                                fmt(test_fnmr_str, -1 * test_max_len)))
+    dev_far_str = "%.3f%%" % (100 * dev_far)
+    test_far_str = "%.3f%%" % (100 * test_far)
+    print("  FAR  | %s | %s" % (fmt(dev_far_str, -1 * dev_max_len),
+                               fmt(test_far_str, -1 * test_max_len)))
+    dev_frr_str = "%.3f%%" % (100 * dev_frr)
+    test_frr_str = "%.3f%%" % (100 * test_frr)
+    print("  FRR  | %s | %s" % (fmt(dev_frr_str, -1 * dev_max_len),
+                               fmt(test_frr_str, -1 * test_max_len)))
+    dev_hter_str = "%.3f%%" % (100 * dev_hter)
+    test_hter_str = "%.3f%%" % (100 * test_hter)
+    print("  HTER | %s | %s" % (fmt(dev_hter_str, -1 * dev_max_len),
+                                fmt(test_hter_str, -1 * test_max_len)))
 
 
 def plots(crit, points, filename, dev_scores, test_scores=None):
@@ -142,7 +159,8 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
   from .. import plot
 
   import matplotlib
-  if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
+  if not hasattr(matplotlib, 'backends'):
+    matplotlib.use('pdf')
   import matplotlib.pyplot as mpl
   from matplotlib.backends.backend_pdf import PdfPages
 
@@ -152,51 +170,53 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
   fig = mpl.figure()
 
   if test_scores is not None:
-    plot.roc(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
-        linestyle='--', dashes=(6,2), label='development')
-    plot.roc(test_neg, test_pos, points, color=(0,0,0),
-        linestyle='-', label='test')
+    plot.roc(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
+             linestyle='--', dashes=(6, 2), label='development')
+    plot.roc(test_neg, test_pos, points, color=(0, 0, 0),
+             linestyle='-', label='test')
   else:
-    plot.roc(dev_neg, dev_pos, points, color=(0,0,0),
-        linestyle='-', label='development')
+    plot.roc(dev_neg, dev_pos, points, color=(0, 0, 0),
+             linestyle='-', label='development')
 
-  mpl.axis([0,40,0,40])
+  mpl.axis([0, 40, 0, 40])
   mpl.title("ROC Curve")
-  mpl.xlabel('FAR (%)')
-  mpl.ylabel('FRR (%)')
-  mpl.grid(True, color=(0.3,0.3,0.3))
-  if test_scores is not None: mpl.legend()
+  mpl.xlabel('FMR (%)')
+  mpl.ylabel('FNMR (%)')
+  mpl.grid(True, color=(0.3, 0.3, 0.3))
+  if test_scores is not None:
+    mpl.legend()
   pp.savefig(fig)
 
   # DET
   fig = mpl.figure()
 
   if test_scores is not None:
-    plot.det(dev_neg, dev_pos, points, color=(0.3,0.3,0.3),
-        linestyle='--', dashes=(6,2), label='development')
-    plot.det(test_neg, test_pos, points, color=(0,0,0),
-        linestyle='-', label='test')
+    plot.det(dev_neg, dev_pos, points, color=(0.3, 0.3, 0.3),
+             linestyle='--', dashes=(6, 2), label='development')
+    plot.det(test_neg, test_pos, points, color=(0, 0, 0),
+             linestyle='-', label='test')
   else:
-    plot.det(dev_neg, dev_pos, points, color=(0,0,0),
-        linestyle='-', label='development')
+    plot.det(dev_neg, dev_pos, points, color=(0, 0, 0),
+             linestyle='-', label='development')
 
   plot.det_axis([0.01, 40, 0.01, 40])
   mpl.title("DET Curve")
-  mpl.xlabel('FAR (%)')
-  mpl.ylabel('FRR (%)')
-  mpl.grid(True, color=(0.3,0.3,0.3))
-  if test_scores is not None: mpl.legend()
+  mpl.xlabel('FMR (%)')
+  mpl.ylabel('FNMR (%)')
+  mpl.grid(True, color=(0.3, 0.3, 0.3))
+  if test_scores is not None:
+    mpl.legend()
   pp.savefig(fig)
 
   # EPC - requires test set
   if test_scores is not None:
     fig = mpl.figure()
     plot.epc(dev_neg, dev_pos, test_neg, test_pos, points,
-        color=(0,0,0), linestyle='-')
+             color=(0, 0, 0), linestyle='-')
     mpl.title('EPC Curve')
     mpl.xlabel('Cost')
     mpl.ylabel('Min. HTER (%)')
-    mpl.grid(True, color=(0.3,0.3,0.3))
+    mpl.grid(True, color=(0.3, 0.3, 0.3))
     pp.savefig(fig)
 
   # Distribution for dev and test scores on the same page
@@ -210,17 +230,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
   fig = mpl.figure()
 
   if test_scores is not None:
-    mpl.subplot(2,1,1)
+    mpl.subplot(2, 1, 1)
     all_scores = numpy.hstack((dev_neg, test_neg, dev_pos, test_pos))
   else:
     all_scores = numpy.hstack((dev_neg, dev_pos))
 
-  nbins=20
+  nbins = 20
   score_range = all_scores.min(), all_scores.max()
   mpl.hist(dev_neg, label='Impostors', normed=True, color='red', alpha=0.5,
-      bins=nbins)
+           bins=nbins)
   mpl.hist(dev_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
-      bins=nbins)
+           bins=nbins)
   mpl.xlim(*score_range)
   _, _, ymax, ymin = mpl.axis()
   mpl.vlines(thres, ymin, ymax, color='black', label='EER', linestyle='dashed')
@@ -229,7 +249,7 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
     ax = mpl.gca()
     ax.axes.get_xaxis().set_ticklabels([])
     mpl.legend(loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.01),
-        fontsize=10)
+               fontsize=10)
     mpl.ylabel('Dev. Scores (normalized)')
   else:
     mpl.ylabel('Normalized Count')
@@ -238,17 +258,17 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
   mpl.grid(True, alpha=0.5)
 
   if test_scores is not None:
-    mpl.subplot(2,1,2)
+    mpl.subplot(2, 1, 2)
     mpl.hist(test_neg, label='Impostors', normed=True, color='red', alpha=0.5,
-        bins=nbins)
+             bins=nbins)
     mpl.hist(test_pos, label='Genuine', normed=True, color='blue', alpha=0.5,
-        bins=nbins)
+             bins=nbins)
     mpl.ylabel('Test Scores (normalized)')
     mpl.xlabel('Score value')
     mpl.xlim(*score_range)
     _, _, ymax, ymin = mpl.axis()
     mpl.vlines(thres, ymin, ymax, color='black', label='EER',
-        linestyle='dashed')
+               linestyle='dashed')
     mpl.grid(True, alpha=0.5)
 
   pp.savefig(fig)
@@ -256,6 +276,28 @@ def plots(crit, points, filename, dev_scores, test_scores=None):
   pp.close()
 
 
+def remove_nan(scores):
+  'removes the NaNs from the scores'
+  nans = numpy.isnan(scores)
+  sum_nans = sum(nans)
+  total = len(scores)
+  if sum_nans > 0:
+    logger.warning('Found {} NaNs in {} scores'.format(sum_nans, total))
+  return scores[numpy.where(~nans)], sum_nans, total
+
+
+def get_fta(scores):
+  'calculates the Failure To Acquire (FtA)'
+  fta_sum, fta_total = 0, 0
+  neg, sum_nans, total = remove_nan(scores[0])
+  fta_sum += sum_nans
+  fta_total += total
+  pos, sum_nans, total = remove_nan(scores[1])
+  fta_sum += sum_nans
+  fta_total += total
+  return ((neg, pos), fta_sum / fta_total)
+
+
 def main(user_input=None):
 
   if user_input is not None:
@@ -269,13 +311,13 @@ def main(user_input=None):
   completions = dict(
       prog=os.path.basename(sys.argv[0]),
       version=pkg_resources.require('bob.measure')[0].version
-      )
+  )
 
   args = docopt.docopt(
       __doc__ % completions,
       argv=argv,
       version=completions['version'],
-      )
+  )
 
   # Sets-up logging
   verbosity = int(args['--verbose'])
@@ -285,23 +327,34 @@ def main(user_input=None):
   try:
     args['--points'] = int(args['--points'])
   except:
-    raise docopt.DocoptExit("cannot convert %s into int for points" % \
-        args['--points'])
+    raise docopt.DocoptExit("cannot convert %s into int for points" %
+                            args['--points'])
 
   if args['--points'] <= 0:
-    raise docopt.DocoptExit('Number of points (--points) should greater ' \
-        'than zero')
+    raise docopt.DocoptExit('Number of points (--points) should greater '
+                            'than zero')
 
-  from ..load import load_score, get_negatives_positives
-  dev_scores = get_negatives_positives(load_score(args['<dev-scores>']))
+  from ..load import get_negatives_positives_from_file
+  dev_scores = get_negatives_positives_from_file(args['<dev-scores>'])
 
   if args['<test-scores>'] is not None:
-    test_scores = get_negatives_positives(load_score(args['<test-scores>']))
+    test_scores = get_negatives_positives_from_file(args['<test-scores>'])
   else:
     test_scores = None
+    test_fta = None
+
+  # test if there are nan in the score files and remove them
+  # also calculate FTA
+  dev_scores, dev_fta = get_fta(dev_scores)
+  print("Failure To Acquire (FTA) in the development set is: {:.3f}%".format(
+     dev_fta * 100))
+  if test_scores is not None:
+    test_scores, test_fta = get_fta(test_scores)
+    print("Failure To Acquire (FTA) in the test set is: {:.3f}%".format(
+       test_fta * 100))
 
-  print_crit('EER', dev_scores, test_scores)
-  print_crit('Min. HTER', dev_scores, test_scores)
+  print_crit('EER', dev_scores, dev_fta, test_scores, test_fta)
+  print_crit('Min. HTER', dev_scores, dev_fta, test_scores, test_fta)
 
   if not args['--no-plot']:
     plots('EER', args['--points'], args['--output'], dev_scores, test_scores)