Merge pull request #12 from bioidiap/DIR

Incorporate Detection and Identification Rate and Curve

bob/measure/load.py

@@ -10,6 +10,9 @@ import numpy
 import tarfile
 import os
 
+import logging
+logger = logging.getLogger('bob.measure')
+
 def open_file(filename, mode='rt'):
   """open_file(filename) -> file_like
 
@@ -121,7 +124,6 @@ def split_four_column(filename):
 
 def cmc_four_column(filename):
   """cmc_four_column(filename) -> cmc_scores
-  
 
   Loads scores to compute CMC curves from a file in four column format.
   The four column file needs to be in the same format as described in :py:func:`four_column`,
@@ -132,7 +134,7 @@ def cmc_four_column(filename):
   Usually, the list of positive scores should contain only one element, but more are allowed.
   The result of this function can directly be passed to, e.g., the :py:func:`bob.measure.cmc` function.
 
-  
+
   **Parameters:**
 
   ``filename`` : str or file-like
@@ -141,48 +143,29 @@ def cmc_four_column(filename):
 
   **Returns:**
 
-  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
-    A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database
+  ``cmc_scores`` : [(negatives, positives)]
+    A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database.
+    Both ``negatives`` and ``positives`` can be either an 1D :py:class:`numpy.ndarray` of type ``float``, or ``None``.
 
   """
   # extract positives and negatives
   pos_dict = {}
   neg_dict = {}
-  # read four column list  
-  for (client_id, probe_id, probe_name, score_str) in four_column(filename):
-    try:
-      score = float(score_str)
-      # check in which dict we have to put the score
-      if client_id == probe_id:
-        correct_dict = pos_dict
-      else:
-        correct_dict = neg_dict
-      # append score
-      if probe_name in correct_dict:
-        correct_dict[probe_name].append(score)
-      else:
-        correct_dict[probe_name] = [score]
-    except:
-      raise SyntaxError("Cannot convert score '%s' to float" % score_str)
-
-  # convert to lists of tuples of ndarrays
-  retval = []
-  import logging
-  logger = logging.getLogger('bob')
-  for probe_name in sorted(pos_dict.keys()):
-    if probe_name in neg_dict:
-      retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
+  # read four column list
+  for (client_id, probe_id, probe_name, score) in four_column(filename):
+    # check in which dict we have to put the score
+    correct_dict = pos_dict if client_id == probe_id else neg_dict
+
+    # append score
+    if probe_name in correct_dict:
+      correct_dict[probe_name].append(score)
     else:
-      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
+      correct_dict[probe_name] = [score]
 
-  #test if there are probes for which only negatives exist
-  for probe_name in sorted(neg_dict.keys()):
-    if not probe_name in pos_dict.keys():
-      logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
+  # convert that into the desired format
+  return _convert_cmc_scores(neg_dict, pos_dict)
 
 
-  return retval
-  
 
 def five_column(filename):
   """five_column(filename) -> claimed_id, model_label, real_id, test_label, score
@@ -259,7 +242,7 @@ def split_five_column(filename):
 
 def cmc_five_column(filename):
   """cmc_four_column(filename) -> cmc_scores
-  
+
   Loads scores to compute CMC curves from a file in five column format.
   The four column file needs to be in the same format as described in :py:func:`five_column`,
   and the ``test_label`` (column 4) has to contain the test/probe file name or a probe id.
@@ -286,35 +269,19 @@ def cmc_five_column(filename):
   # read four column list
   for (client_id, _, probe_id, probe_name, score) in five_column(filename):
     # check in which dict we have to put the score
-    if client_id == probe_id:
-      correct_dict = pos_dict
-    else:
-      correct_dict = neg_dict
+    correct_dict = pos_dict if client_id == probe_id else neg_dict
+
     # append score
     if probe_name in correct_dict:
       correct_dict[probe_name].append(score)
     else:
       correct_dict[probe_name] = [score]
 
-  # convert to lists of tuples of ndarrays
-  retval = []
-  import logging
-  logger = logging.getLogger('bob')
-
-  for probe_name in sorted(pos_dict.keys()):
-    if probe_name in neg_dict:
-      retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
-    else:
-      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
-  # test if there are probes for which only negatives exist
-  for probe_name in sorted(neg_dict.keys()):
-    if not probe_name in pos_dict.keys():
-       logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
+  # convert that into the desired format
+  return _convert_cmc_scores(neg_dict, pos_dict)
 
-  return retval
 
-
-def load_score(filename, ncolumns=None):
+def load_score(filename, ncolumns = 4):
   """Load scores using numpy.loadtxt and return the data as a numpy array.
 
   **Parameters:**
@@ -333,11 +300,8 @@ def load_score(filename, ncolumns=None):
     'claimed_id', 'real_id', 'test_label', and ['model_label']
 
   """
-  if ncolumns is None:
-    ncolumns = 4
 
-  def convertfunc(x):
-    return x
+  convertfunc = lambda x : x
 
   if ncolumns == 4:
     names = ('claimed_id', 'real_id', 'test_label', 'score')
@@ -410,3 +374,13 @@ def dump_score(filename, score_lines):
   else:
     raise ValueError("Only scores with 4 and 5 columns are supported.")
   numpy.savetxt(filename, score_lines, fmt=fmt)
+
+def _convert_cmc_scores(neg_dict, pos_dict):
+  """Converts the negative and positive scores read with :py:func:`cmc_four_column` or :py:func:`cmc_four_column` into a format that is handled by the :py:func:`bob.measure.cmc` and similar functions."""
+  # convert to lists of tuples of ndarrays (or None)
+  probe_names = sorted(set(neg_dict.keys()).union(set(pos_dict.keys())))
+  # get all scores in the desired format
+  return [(
+    numpy.array(neg_dict[probe_name], numpy.float64) if probe_name in neg_dict else None,
+    numpy.array(pos_dict[probe_name], numpy.float64) if probe_name in pos_dict else None
+  ) for probe_name in probe_names]

bob/measure/plot.py

@@ -3,10 +3,34 @@
 # Chakka Murali Mohan, Trainee, IDIAP Research Institute, Switzerland.
 # Mon 23 May 2011 14:36:14 CEST
 
+def log_values(min_step = -4, counts_per_step = 4):
+  """log_values(min_step, counts_per_step) -> log_list
+
+  This function computes log-scaled values between :math:`10^{M}` and 1 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a negative integer.
+  The integral ``counts_per_step`` value defines how many values between two adjacent powers of 10 will be created.
+  The total number of values will be ``-min_step * counts_per_step + 1``.
+
+  **Parameters:**
+
+  ``min_step`` : int (negative)
+    The power of 10 that will be the minimum value. E.g., the default ``-4`` will result in the first number to be :math:`10^{-4}` = ``0.00001`` or ``0.01%``
+
+  ``counts_per_step`` : int (positive)
+    The number of values that will be put between two adjacent powers of 10.
+    With the default value ``4`` (and default values of ``min_step``), we will get ``log_list[0] == 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``.
+
+  **Returns**
+
+  ``log_list`` : [float]
+    A list of logarithmically scaled values between :math:`10^{M}` and 1.
+  """
+  import math
+  return [math.pow(10., i * 1./counts_per_step) for i in range(min_step*counts_per_step,0)] + [1.]
+
 """Methods to plot error analysis figures such as ROC, precision-recall curve, EPC and DET"""
 
 def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
-  """Plots Receiver Operating Charactaristic (ROC) curve.
+  """Plots Receiver Operating Characteristic (ROC) curve.
 
   This method will call ``matplotlib`` to plot the ROC curve for a system which
   contains a particular set of negatives (impostors) and positives (clients)
@@ -51,6 +75,46 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
     return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
 
 
+def roc_for_far(negatives, positives, far_values = log_values(), **kwargs):
+  """Plots Receiver Operating Characteristic (ROC) curve for the given list of False Acceptance Rates (FAR).
+
+  This method will call ``matplotlib`` to plot the ROC curve for a system which
+  contains a particular set of negatives (impostors) and positives (clients)
+  scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. All parameters
+  passed with exception of the three first parameters of this method will be
+  directly passed to the plot command.
+
+  The plot will represent the False Acceptance Rate (FAR) on the horizontal axis and the Correct Acceptance Rate (CAR) on the vertical axis.
+  The values for the axis will be computed using :py:func:`bob.measure.roc_for_far`.
+
+  .. note::
+
+    This function does not initiate and save the figure instance, it only
+    issues the plotting command. You are the responsible for setting up and
+    saving the figure as you see fit.
+
+  **Parameters:**
+
+  ``negatives, positives`` : array_like(1D, float)
+    The list of negative and positive scores forwarded to :py:func:`bob.measure.roc`
+
+  ``far_values`` : [float]
+    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
+
+  ``kwargs`` : keyword arguments
+    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
+
+  **Returns:**
+
+  The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.semilogx`.
+  """
+
+  from matplotlib import pyplot
+  from . import roc_for_far as calc
+  out = calc(negatives, positives, far_values)
+  return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
+
+
 def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
   """Plots Precision-Recall curve.
 
@@ -341,3 +405,59 @@ def cmc(cmc_scores, logx = True, **kwargs):
     pyplot.plot(range(1, len(out)+1), out * 100, **kwargs)
 
   return len(out)
+
+
+def detection_identification_curve(cmc_scores, far_values = log_values(), rank = 1, logx = True, **kwargs):
+  """Plots the Detection & Identification curve over the FAR for the given FAR values.
+  This curve is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
+  It requires to have at least one open set probe item, i.e., with no corresponding gallery, such that the positives for that pair are ``None``.
+
+  The detection and identification curve first computes FAR thresholds based on the out-of-set probe scores (negative scores).
+  For each probe item, the **maximum** negative score is used.
+  Then, it plots the detection and identification rates for those thresholds, which are based on the in-set probe scores only.
+  See [LiJain2005]_ for more details.
+
+  **Parameters:**
+
+  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
+    See :py:func:`bob.measure.detection_identification_rate`
+
+  ``far_values`` : [float]
+    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
+
+  ``rank`` : int or ``None``
+    The rank for which the curve should be plotted, 1 by default.
+
+  ``logx`` : bool
+    Plot the FAR axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``)
+
+  ``kwargs`` : keyword arguments
+    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.
+
+  **Returns:**
+
+  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
+
+  .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
+  """
+
+  import numpy
+  from matplotlib import pyplot
+  from . import far_threshold, detection_identification_rate
+
+  # for each probe, for which no positives exists, get the highest negative score; and sort them to compute the FAR thresholds
+  negatives = sorted(max(neg) for neg,pos in cmc_scores if (pos is None or not numpy.array(pos).size) and neg is not None)
+  if not negatives:
+    raise ValueError("There need to be at least one pair with only negative scores")
+
+  # compute thresholds based on FAR values
+  thresholds = [far_threshold(negatives, [], v, True) for v in far_values]
+
+  # compute detection and identification rate based on the thresholds for the given rank
+  rates = [100.*detection_identification_rate(cmc_scores, t, rank) for t in thresholds]
+
+  # plot curve
+  if logx:
+    return pyplot.semilogx(far_values, rates, **kwargs)
+  else:
+    return pyplot.plot(far_values, rates, **kwargs)

bob/measure/script/plot_cmc.py

@@ -30,6 +30,7 @@ def parse_command_line(command_line_options):
   parser.add_argument('-s', '--score-file', required = True, help = 'The score file in 4 or 5 column format to test.')
   parser.add_argument('-o', '--output-pdf-file', default = 'cmc.pdf', help = 'The PDF file to write.')
   parser.add_argument('-l', '--log-x-scale', action='store_true', help = 'Plot logarithmic Rank axis.')
+  parser.add_argument('-r', '--rank', type=int, help = 'Plot Detection & Identification rate curve for the given rank instead of the CMC curve.')
   parser.add_argument('-x', '--no-plot', action = 'store_true', help = 'Do not print a PDF file, but only report the results.')
   parser.add_argument('-p', '--parser', default = '4column', choices = ('4column', '5column'), help = 'The type of the score file.')
 
@@ -57,7 +58,7 @@ def main(command_line_options = None):
   data = {'4column' : load.cmc_four_column, '5column' : load.cmc_five_column}[args.parser](args.score_file)
 
   # compute recognition rate
-  rr = recognition_rate(data)
+  rr = recognition_rate(data, args.rank)
   print("Recognition rate for score file", args.score_file, "is %3.2f%%" % (rr * 100))
 
   if not args.no_plot:
@@ -71,19 +72,34 @@ def main(command_line_options = None):
 
     # CMC
     fig = mpl.figure()
-    max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
-    mpl.title("CMC Curve")
-    if args.log_x_scale:
-      mpl.xlabel('Rank (log)')
+    if args.rank is None:
+      max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
+      mpl.title("CMC Curve")
+      if args.log_x_scale:
+        mpl.xlabel('Rank (log)')
+      else:
+        mpl.xlabel('Rank')
+      mpl.ylabel('Recognition Rate in %')
+
+      ticks = [int(t) for t in mpl.xticks()[0]]
+      mpl.xticks(ticks, ticks)
+      mpl.xlim([1, max_rank])
     else:
-      mpl.xlabel('Rank')
-    mpl.ylabel('Recognition Rate in %')
+      plot.detection_identification_curve(data, rank = args.rank, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
+      mpl.title("Detection & Identification Curve")
+      if args.log_x_scale:
+        mpl.xlabel('False Acceptance Rate (log) in %')
+      else:
+        mpl.xlabel('False Acceptance Rate in %')
+      mpl.ylabel('Detection & Identification Rate in %')
+
+      ticks = ["%s"%(t*100) for t in mpl.xticks()[0]]
+      mpl.xticks(mpl.xticks()[0], ticks)
+      mpl.xlim([1e-4, 1])
+
     mpl.grid(True, color=(0.3,0.3,0.3))
     mpl.ylim(ymax=101)
     # convert log-scale ticks to normal numbers
-    ticks = [int(t) for t in mpl.xticks()[0]]
-    mpl.xticks(ticks, ticks)
-    mpl.xlim([0.9, max_rank + 0.1])
 
     pp.savefig(fig)
     pp.close()

bob/measure/test_error.py

@@ -270,7 +270,7 @@ def test_cmc():
 
   # tests the CMC calculation
   # test data; should give match characteristics [1/2,1/4,1/3] and CMC [1/3,2/3,1]
-  test_data = [((0.3, 1.1, 0.5), (0.7)), ((1.4, -1.3, 0.6), (0.2)), ((0.8, 0., 1.5), (-0.8, 1.8)), ((2., 1.3, 1.6, 0.9), (2.4))]
+  test_data = [((0.3, 1.1, 0.5), (0.7,)), ((1.4, -1.3, 0.6), (0.2,)), ((0.8, 0., 1.5), (-0.8, 1.8)), ((2., 1.3, 1.6, 0.9), (2.4,))]
   # compute recognition rate
   rr = recognition_rate(test_data)
   nose.tools.eq_(rr, 0.5)
@@ -319,36 +319,34 @@ def test_calibration():
   min_cllr = calibration.min_cllr(negatives, positives)
 
   assert min_cllr <= cllr
-  assert abs(cllr - 3.6183) < 1e-4
-  assert abs(min_cllr - 0.3373) < 1e-4
+  assert abs(cllr - 3.61833) < 1e-5, cllr
+  assert abs(min_cllr - 0.33736) < 1e-5, min_cllr
 
 
 
-def test_open_set_recognition_rate():
-
-  far_value = 0.01
+def test_open_set_rates():
 
   # No error files
   cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"))
-  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set.txt"))
+  assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 1.0) < 1e-8
+  assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5)) < 1e-8
 
-  assert abs(bob.measure.recognition_rate(cmc_scores) - 1.0) < 1e-8
+  assert abs(bob.measure.recognition_rate(cmc_scores) - 7./9.) < 1e-8
   assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 1.0) < 1e-8
-  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
-  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t) - 1.0) < 1e-8
 
   # One error
   cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-one-error.txt"))
-  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set-one-error.txt"))
-  assert abs(bob.measure.recognition_rate(cmc_scores) - 0.857142857143) < 1e-8
-  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 0.857142857143) < 1e-8
-  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
-  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t) - 0.857142857143) < 1e-8
+  assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
+  assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5)) < 1e-8
+
+  assert abs(bob.measure.recognition_rate(cmc_scores) - 6./9.) < 1e-8
+  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
+
 
   # Two errors
   cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-two-errors.txt"))
-  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set-two-errors.txt"))
-  assert abs(bob.measure.recognition_rate(cmc_scores) - 0.857142857143) < 1e-8
-  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 0.857142857143) < 1e-8
-  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
-  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t)) < 1e-8
+  assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
+  assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5) - 0.5) < 1e-8
+
+  assert abs(bob.measure.recognition_rate(cmc_scores) - 6./9.) < 1e-8
+  assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 6./8.) < 1e-8

bob/measure/test_scripts.py

@@ -25,6 +25,8 @@ TEST_SCORES_5COL = F('test-5col.txt')
 SCORES_4COL_CMC = F('scores-cmc-4col.txt')
 SCORES_5COL_CMC = F('scores-cmc-5col.txt')
 
+SCORES_4COL_CMC_OS = F('scores-cmc-4col-open-set.txt')
+
 def test_compute_perf():
 
   # sanity checks
@@ -68,7 +70,9 @@ def test_compute_cmc():
   # sanity checks
   assert os.path.exists(SCORES_4COL_CMC)
   assert os.path.exists(SCORES_5COL_CMC)
+  assert os.path.exists(SCORES_4COL_CMC_OS)
 
   from .script.plot_cmc import main
   nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC, '--log-x-scale']), 0)
   nose.tools.eq_(main(['--self-test', '--score-file', SCORES_5COL_CMC, '--parser', '5column']), 0)
+  nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC_OS, '--rank', '1']), 0)

doc/guide.rst

@@ -19,7 +19,8 @@
 Methods in the :py:mod:`bob.measure` module can help you to quickly and easily
 evaluate error for multi-class or binary classification problems. If you are
 not yet familiarized with aspects of performance evaluation, we recommend the
-following papers for an overview of some of the methods implemented.
+following papers and book chapters for an overview of some of the implemented
+methods.
 
 * Bengio, S., Keller, M., Mariéthoz, J. (2004). `The Expected Performance
   Curve`_.  International Conference on Machine Learning ICML Workshop on ROC
@@ -27,6 +28,8 @@ following papers for an overview of some of the methods implemented.
 * Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
   `The DET curve in assessment of detection task performance`_. Fifth European
   Conference on Speech Communication and Technology (pp. 1895-1898).
+* Li, S., Jain, A.K. (2005), `Handbook of Face Recognition`, Chapter 14, Springer
+
 
 Overview
 --------
@@ -105,8 +108,8 @@ defined in the first equation.
   loaded your scores in two 1D float64 vectors and are ready to evaluate the
   performance of the classifier.
 
-Evaluation
-----------
+Verification
+------------
 
 To count the number of correctly classified positives and negatives you can use
 the following techniques:
@@ -171,6 +174,40 @@ calculation of the threshold:
       >>> t = bob.measure.min_weighted_error_rate_threshold(negatives, positives, cost, is_sorted = True)
       >>> assert T == t
 
+Identification
+--------------
+
+For identification, the Recognition Rate is one of the standard measures.
+To compute recognition rates, you can use the :py:func:`bob.measure.recognition_rate` function.
+This function expects a relatively complex data structure, which is the same as for the `CMC`_ below.
+For each probe item, the scores for negative and positive comparisons are computed, and collected for all probe items:
+
+.. doctest::
+
+   >>> rr_scores = []
+   >>> for probe in range(10):
+   ...   pos = numpy.random.normal(1, 1, 1)
+   ...   neg = numpy.random.normal(0, 1, 19)
+   ...   rr_scores.append((neg, pos))
+   >>> rr = bob.measure.recognition_rate(rr_scores, rank=1)
+
+For open set identification, according to Li and Jain (2005) there are two different error measures defined.
+The first measure is the :py:func:`bob.measure.detection_identification_rate`, which counts the number of correctly classified in-gallery probe items.
+The second measure is the :py:func:`bob.measure.false_alarm_rate`, which counts, how often an out-of-gallery probe item was incorrectly accepted.
+Both rates can be computed using the same data structure, with one exception.
+Both functions require that at least one probe item exists, which has no according gallery item, i.e., where the positives are empty or ``None``:
+
+(continued from above...)
+
+.. doctest::
+
+   >>> for probe in range(10):
+   ...   pos = None
+   ...   neg = numpy.random.normal(-2, 1, 10)
+   ...   rr_scores.append((neg, pos))
+   >>> dir = bob.measure.detection_identification_rate(rr_scores, threshold = 0, rank=1)
+   >>> far = bob.measure.false_alarm_rate(rr_scores, threshold = 0)
+
 
 Plotting
 --------
@@ -202,6 +239,7 @@ You should see an image like the following one:
 .. plot::
 
    import numpy
+   numpy.random.seed(42)
    import bob.measure
    from matplotlib import pyplot
 
@@ -215,13 +253,13 @@ You should see an image like the following one:
    pyplot.title('ROC')
 
 As can be observed, plotting methods live in the namespace
-:py:mod:`bob.measure.plot`. They work like `Matplotlib`_'s `plot()`_ method
+:py:mod:`bob.measure.plot`. They work like the :py:func:`matplotlib.pyplot.plot`
 itself, except that instead of receiving the x and y point coordinates as
 parameters, they receive the two :py:class:`numpy.ndarray` arrays with
 negatives and positives, as well as an indication of the number of points the
 curve must contain.
 
-As in `Matplotlib`_'s `plot()`_ command, you can pass optional parameters for
+As in the :py:func:`matplotlib.pyplot.plot` command, you can pass optional parameters for
 the line as shown in the example to setup its color, shape and even the label.
 For an overview of the keywords accepted, please refer to the `Matplotlib`_'s
 Documentation. Other plot properties such as the plot title, axis labels,
@@ -250,6 +288,7 @@ This will produce an image like the following one:
 .. plot::
 
    import numpy
+   numpy.random.seed(42)
    import bob.measure
    from matplotlib import pyplot
 
@@ -300,6 +339,7 @@ This will produce an image like the following one:
 .. plot::
 
    import numpy
+   numpy.random.seed(42)
    import bob.measure
    from matplotlib import pyplot
 
@@ -323,26 +363,59 @@ The CMC can be calculated from a relatively complex data structure, which define
 .. plot::
 
    import numpy
+   numpy.random.seed(42)
    import bob.measure
    from matplotlib import pyplot
 
-   scores = []
+   cmc_scores = []
    for probe in range(10):
      positives = numpy.random.normal(1, 1, 1)
      negatives = numpy.random.normal(0, 1, 19)
-     scores.append((negatives, positives))
-   bob.measure.plot.cmc(scores, logx=False)
+     cmc_scores.append((negatives, positives))
+   bob.measure.plot.cmc(cmc_scores, logx=False)
    pyplot.title('CMC')
    pyplot.xlabel('Rank')
    pyplot.xticks([1,5,10,20])
    pyplot.xlim([1,20])
    pyplot.ylim([0,100])
+   pyplot.ylabel('Probability of Recognition (%)')
 
 Usually, there is only a single positive score per probe, but this is not a fixed restriction.
 
 .. note::
    The complex data structure can be read from our default 4 or 5 column score files using the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function.
 
+
+Detection & Identification Curve
+================================
+
+The detection & identification curve is designed to evaluate open set identification tasks.
+It can be plotted using the :py:func:`bob.measure.plot.detection_identification_curve` function, but it requires at least one open-set probe, i.e., where no corresponding positive score exists, for which the FAR values are computed.
+Here, we plot the detection and identification curve for rank 1, so that the recognition rate for FAR=1 will be identical to the rank one :py:func:`bob.measure.recognition_rate` obtained in the CMC plot above.
+
+.. plot::
+
+   import numpy
+   numpy.random.seed(42)
+   import bob.measure
+   from matplotlib import pyplot
+
+   cmc_scores = []
+   for probe in range(10):
+     positives = numpy.random.normal(1, 1, 1)
+     negatives = numpy.random.normal(0, 1, 19)
+     cmc_scores.append((negatives, positives))
+   for probe in range(10):
+     negatives = numpy.random.normal(-1, 1, 10)
+     cmc_scores.append((negatives, None))
+
+   bob.measure.plot.detection_identification_curve(cmc_scores, rank=1, logx=True)
+   pyplot.xlabel('False Alarm Rate')
+   pyplot.ylabel('Detection & Identification Rate (%)')
+   pyplot.ylim([0,100])
+
+
+
 Fine-tunning
 ============
 
@@ -497,5 +570,4 @@ These information are simply stored in the score file, and no further check is a
 
 .. _`The Expected Performance Curve`: http://publications.idiap.ch/downloads/reports/2005/bengio_2005_icml.pdf
 .. _`The DET curve in assessment of detection task performance`: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf
-.. _`plot()`: http://matplotlib.sourceforge.net/api/pyplot_api.html#matplotlib.pyplot.plot
 .. _openbr: http://openbiometrics.org

doc/py_api.rst

@@ -28,6 +28,8 @@ Single point measurements
    bob.measure.f_score
    bob.measure.precision_recall
    bob.measure.recognition_rate
+   bob.measure.detection_identification_rate
+   bob.measure.false_alarm_rate
    bob.measure.eer_rocch
 
 Thresholds
@@ -92,6 +94,7 @@ Plotting
    bob.measure.plot.epc
    bob.measure.plot.precision_recall_curve
    bob.measure.plot.cmc
+   bob.measure.plot.detection_identification_curve
 
 OpenBR conversions
 ------------------