Merge pull request #12 from bioidiap/DIR

Incorporate Detection and Identification Rate and Curve

bob/measure/__init__.py

@@ -79,62 +79,108 @@ def relevance (input, machine):
   return retval
 
 
-def recognition_rate(cmc_scores, threshold=None):
-  """recognition_rate(cmc_scores, threshold) -> RR 
-  
+def recognition_rate(cmc_scores, threshold = None, rank = 1):
+  """recognition_rate(cmc_scores, rank, threshold) -> RR
+
   Calculates the recognition rate from the given input, which is identical
-  to the rank 1 (C)MC value.
+  to the CMC value for the given ``rank``.
 
-  The input has a specific format, which is a list of two-element tuples. Each
-  of the tuples contains the negative and the positive scores for one test
-  item.  To read the lists from score files in 4 or 5 column format, please use
-  the :py:func:`bob.measure.load.cmc_four_column` or
-  :py:func:`bob.measure.load.cmc_five_column` function.
+  The input has a specific format, which is a list of two-element tuples.
+  Each of the tuples contains the negative :math:`\\{S_p^-\\}` and the positive :math:`\\{S_p^+\\}` scores for one probe item :math:`p`, or ``None`` in case of open set recognition.
+  To read the lists from score files in 4 or 5 column format, please use the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function.
+
+  If **threshold** is set to ``None``, the rank 1 recognition rate is defined as the number of test items, for which the highest positive :math:`\\max\\{S_p^+\\}` score is greater than or equal to all negative scores, divided by the number of all probe items :math:`P`:
+
+  .. math::
 
-  If **threshold** is set to `None`, the recognition rate is defined as the number of test items, for which the
-  positive score is greater than or equal to all negative scores, divided by
-  the number of all test items. If several positive scores for one test item exist, the **highest** score is taken.
+    \\mathrm{RR} = \\frac{1}{P} \\sum_{p=1}^{P} \\begin{cases} 1 & \\mathrm{if } \\max\\{S_p^+\\} >= \\max\\{S_p^-\\}\\\\ 0 & \\mathrm{otherwise} \\end{cases}
+
+  For a given rank :math:`r>1`, up to :math:`r` negative scores that are higher than the highest positive score are allowed to still count as correctly classified in the top :math:`r` rank.
+
+  If ``threshold`` :math:`\\theta` is given, **all** scores below threshold will be filtered out.
+  Hence, if all positive scores are below threshold :math:`\\max\\{S_p^+\\} < \\theta`, the probe will be misclassified **at any rank**.
+
+  For open set recognition, i.e., when there exist a tuple including negative scores without corresponding positive scores (``None``), and **all** negative scores are below ``threshold`` :math:`\\max\\{S_p^+\\} < \\theta`, the probe item is correctly rejected, **and it does not count into the denominator** :math:`P`.
+  When no ``threshold`` is provided, the open set probes will **always** count as misclassified, regardless of the ``rank``.
+
+  .. warn:
+     For open set tests, this rate does not correspond to a standard rate.
+     Please use :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate` instead.
 
-  If **threshold** assumes one value, the recognition rate is defined as the number of test items, for which the
-  positive score is greater than or equal to all negative scores and the threshold divided by
-  the number of all test items. If several positive scores for one test item exist, the **highest** score is taken.
-  
   **Parameters:**
-  
-    ``cmc_scores`` : CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`)
 
-    ``threshold`` : Decision threshold. If `None`, the decision threshold will be the **highest** positive score.
-    
+  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
+    CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`).
+    Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**.
+    Each pair can contain up to one empty array (or ``None``), i.e., in case of open set recognition.
+
+  ``threshold`` : float or ``None``
+    Decision threshold. If not ``None``, **all** scores will be filtered by the threshold.
+    In an open set recognition problem, all open set scores (negatives with no corresponding positive) for which all scores are below threshold, will be counted as correctly rejected and **removed** from the probe list (i.e., the denominator).
+
+  ``rank`` : int or ``None``
+    The rank for which the recognition rate should be computed, 1 by default.
+
   **Returns:**
 
   ``RR`` : float
-    The rank 1 recognition rate, i.e., the relative number of correctly identified identities
+    The (open set) recognition rate for the given rank, a value between 0 and 1.
   """
   # If no scores are given, the recognition rate is exactly 0.
   if not cmc_scores:
     return 0.
 
-  correct = 0.
+  correct = 0
+  counter = 0
   for neg, pos in cmc_scores:
+    # set all values that are empty before to None
+    if pos is not None and not numpy.array(pos).size:
+      pos = None
+    if neg is not None and not numpy.array(neg).size:
+      neg = None
+
+    if pos is None and neg is None:
+      raise ValueError("One pair of the CMC scores has neither positive nor negative values")
+
+    # filter out any negative or positive scores below threshold; scores with exactly the threshold are also filtered out
+    # now, None and an empty array have different meanings.
+    if threshold is not None:
+      if neg is not None:
+        neg = numpy.array(neg)[neg > threshold]
+      if pos is not None:
+        pos = numpy.array(pos)[pos > threshold]
+
+    if pos is None:
+      # no positives, so we definitely do not have a match;
+      # check if we have negatives above threshold
+      if not neg.size:
+        # we have no negative scores over the threshold, so we have correctly rejected the probe
+        # don't increase any of the two counters...
+        continue
+      # we have negatives over threshold, so we have incorrect classifications; independent on the actual rank
+      counter += 1
+    else:
+      # we have a positive, so we need to count the probe
+      counter += 1
+
+      if not numpy.array(pos).size:
+        # all positive scores have been filtered out by the threshold, we definitely have a mis-match
+        continue
 
-    #If threshold is none, let's use the highest positive score as the decision threshold
-    if(threshold is None):
       # get the maximum positive score for the current probe item
       # (usually, there is only one positive score, but just in case...)
       max_pos = numpy.max(pos)
-      # check if the positive score is smaller than all negative scores
-      if (neg < max_pos).all():
-        correct += 1.
-        
-    else:
-      #If threshold is NOT None, we have an openset identification
-      max_pos = numpy.max(pos)
 
-      if((threshold < max_pos) and (neg < max_pos).all()):
-          correct += 1.
-  # return relative number of correctly matched scores
-  return correct / float(len(cmc_scores))
+      if neg is None or not numpy.array(neg).size:
+        # if we had no negatives, or all negatives were below threshold, we have a match at rank 1
+        correct += 1
+      else:
+        # count the number of negative scores that are higher than the best positive score
+        index = numpy.sum(neg >= max_pos)
+        if index < rank:
+          correct += 1
 
+  return float(correct) / float(counter)
 
 
 def cmc(cmc_scores):
@@ -143,24 +189,26 @@ def cmc(cmc_scores):
   Calculates the cumulative match characteristic (CMC) from the given input.
 
   The input has a specific format, which is a list of two-element tuples. Each
-  of the tuples contains the negative and the positive scores for one test
+  of the tuples contains the negative and the positive scores for one probe
   item.  To read the lists from score files in 4 or 5 column format, please use
   the :py:func:`bob.measure.load.cmc_four_column` or
   :py:func:`bob.measure.load.cmc_five_column` function.
 
-  For each test item the probability that the rank r of the positive score is
+  For each probe item the probability that the rank :math:`r` of the positive score is
   calculated.  The rank is computed as the number of negative scores that are
   higher than the positive score.  If several positive scores for one test item
   exist, the **highest** positive score is taken. The CMC finally computes how
-  many test items have rank r or higher.
+  many test items have rank r or higher, divided by the total number of test values.
+
+  .. note::
+    The CMC is not available for open set classification.
+    Please use the :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate` instead.
 
   **Parameters:**
 
   ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
     A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database
 
-  ``threshold`` : Decision threshold. If `None`, the decision threshold will be the **highest** positive score.
-
   **Returns:**
 
   ``curve`` : array_like(2D, float)
@@ -173,29 +221,124 @@ def cmc(cmc_scores):
     raise ValueError("The given set of scores is empty")
 
   # compute MC
-  match_characteristic = numpy.zeros((max([len(neg) for (neg,pos) in cmc_scores])+1,), numpy.int)
+  match_characteristic = numpy.zeros((max([len(neg) for neg, _ in cmc_scores if neg is not None])+1,), numpy.int)
 
   for neg, pos in cmc_scores:
-    if((type(pos)!=float) and (len(pos) == 0)):
-      raise ValueError("For the CMC computation at least one positive score is necessary. Please review who you are loading the scores. You must set `load_only_negatives=False` in the :py:func:`bob.measure.load.cmc_four_column` or `:py:func:`bob.measure.load.cmc_five_column` methods.")
+    if pos is None or not numpy.array(pos).size:
+      raise ValueError("For the CMC computation at least one positive score per pair is necessary.")
+    if neg is None:
+      neg = []
 
     # get the maximum positive score for the current probe item
-    # (usually, there is only one positive score, but just in case...)    
+    # (usually, there is only one positive score, but just in case...)
     max_pos = numpy.max(pos)
 
-    # count the number of negative scores that are higher than the best positive score            
+    # count the number of negative scores that are higher than the best positive score
     index = numpy.sum(neg >= max_pos)
-    match_characteristic[index] += 1  
-    
+    match_characteristic[index] += 1
+
   # cumulate
-  cumulative_match_characteristic = numpy.ndarray(match_characteristic.shape, numpy.float64)
-  count = 0.
-  for i in range(match_characteristic.shape[0]):
-    count += match_characteristic[i]
-    cumulative_match_characteristic[i] = count / probe_count
+  cumulative_match_characteristic = numpy.cumsum(match_characteristic, dtype=numpy.float64)
+  return cumulative_match_characteristic / probe_count
 
-  return cumulative_match_characteristic
 
+def detection_identification_rate(cmc_scores, threshold, rank = 1):
+  """detection_identification_rate(cmc_scores, threshold, rank) -> dir
+
+  Computes the `detection and identification rate` for the given threshold.
+  This value is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
+
+  Although the detection and identification rate is designed to be computed on an open set protocol, it uses only the probe elements, for which a corresponding gallery element exists.
+  For closed set identification protocols, this function is identical to :py:func:`recognition_rate`.
+  The only difference is that for this function, a ``threshold`` for the scores need to be defined, while for :py:func:`recognition_rate` it is optional.
+
+  **Parameters:**
+
+  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
+    CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`).
+    Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**.
+    There need to be at least one probe item, for which positive and negative scores exist.
+
+  ``threshold`` : float
+    The decision threshold :math:`\\tau``.
+
+  ``rank`` : int
+    The rank for which the curve should be plotted, by default 1.
+
+  **Returns:**
+
+  ``dir`` : float
+    The detection and identification rate for the given threshold.
+  """
+  # count the correctly classifier probes
+  correct = 0
+  counter = 0
+  for neg, pos in cmc_scores:
+    if pos is None or not numpy.array(pos).size:
+      # we only consider probes with corresponding gallery items
+      continue
+    # we have an in-gallery probe
+    counter += 1
+    # check, if it is correctly classified
+    if neg is None:
+      neg = []
+
+    # get the maximum positive score for the current probe item
+    # (usually, there is only one positive score, but just in case...)
+    max_pos = numpy.max(pos)
+
+    index = numpy.sum(neg >= max_pos) # compute the rank (in fact, rank - 1)
+    if max_pos >= threshold and index < rank:
+      correct += 1
+
+  if not counter:
+    logger.warn("No in-gallery probe was found")
+    return 0.
+
+  return float(correct) / float(counter)
+
+
+def false_alarm_rate(cmc_scores, threshold):
+  """false_alarm_rate(cmc_scores, threshold) -> far
+
+  Computes the `false alarm rate` for the given threshold,.
+  This value is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
+
+  The false alarm rate is designed to be computed on an open set protocol, it uses only the probe elements, for which **no** corresponding gallery element exists.
+
+  **Parameters:**
+
+  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
+    CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`).
+    Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**.
+    There need to be at least one probe item, for which only negative scores exist.
+
+  ``threshold`` : float
+    The decision threshold :math:`\\tau``.
+
+  **Returns:**
+
+  ``far`` : float
+    The false alarm rate.
+  """
+  incorrect = 0
+  counter = 0
+  for neg, pos in cmc_scores:
+    # we only consider the out-of-gallery probes, i.e., with no positive scores
+    if pos is None or not numpy.array(pos).size:
+      counter += 1
+
+      # check if the probe is above threshold
+      if neg is None or not numpy.array(neg).size:
+        raise ValueError("One pair of the CMC scores has neither positive nor negative values")
+      if numpy.max(neg) >= threshold:
+        incorrect += 1
+
+  if not counter:
+    logger.warn("No out-of-gallery probe was found")
+    return 0.
+
+  return float(incorrect) / float(counter)
 
 
 def get_config():

bob/measure/load.py

@@ -10,6 +10,9 @@ import numpy
 import tarfile
 import os
 
+import logging
+logger = logging.getLogger('bob.measure')
+
 def open_file(filename, mode='rt'):
   """open_file(filename) -> file_like
 
@@ -121,7 +124,6 @@ def split_four_column(filename):
 
 def cmc_four_column(filename):
   """cmc_four_column(filename) -> cmc_scores
-  
 
   Loads scores to compute CMC curves from a file in four column format.
   The four column file needs to be in the same format as described in :py:func:`four_column`,
@@ -132,7 +134,7 @@ def cmc_four_column(filename):
   Usually, the list of positive scores should contain only one element, but more are allowed.
   The result of this function can directly be passed to, e.g., the :py:func:`bob.measure.cmc` function.
 
-  
+
   **Parameters:**
 
   ``filename`` : str or file-like
@@ -141,48 +143,29 @@ def cmc_four_column(filename):
 
   **Returns:**
 
-  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
-    A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database
+  ``cmc_scores`` : [(negatives, positives)]
+    A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database.
+    Both ``negatives`` and ``positives`` can be either an 1D :py:class:`numpy.ndarray` of type ``float``, or ``None``.
 
   """
   # extract positives and negatives
   pos_dict = {}
   neg_dict = {}
-  # read four column list  
-  for (client_id, probe_id, probe_name, score_str) in four_column(filename):
-    try:
-      score = float(score_str)
-      # check in which dict we have to put the score
-      if client_id == probe_id:
-        correct_dict = pos_dict
-      else:
-        correct_dict = neg_dict
-      # append score
-      if probe_name in correct_dict:
-        correct_dict[probe_name].append(score)
-      else:
-        correct_dict[probe_name] = [score]
-    except:
-      raise SyntaxError("Cannot convert score '%s' to float" % score_str)
-
-  # convert to lists of tuples of ndarrays
-  retval = []
-  import logging
-  logger = logging.getLogger('bob')
-  for probe_name in sorted(pos_dict.keys()):
-    if probe_name in neg_dict:
-      retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
+  # read four column list
+  for (client_id, probe_id, probe_name, score) in four_column(filename):
+    # check in which dict we have to put the score
+    correct_dict = pos_dict if client_id == probe_id else neg_dict
+
+    # append score
+    if probe_name in correct_dict:
+      correct_dict[probe_name].append(score)
     else:
-      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
+      correct_dict[probe_name] = [score]
 
-  #test if there are probes for which only negatives exist
-  for probe_name in sorted(neg_dict.keys()):
-    if not probe_name in pos_dict.keys():
-      logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
+  # convert that into the desired format
+  return _convert_cmc_scores(neg_dict, pos_dict)
 
 
-  return retval
-  
 
 def five_column(filename):
   """five_column(filename) -> claimed_id, model_label, real_id, test_label, score
@@ -259,7 +242,7 @@ def split_five_column(filename):
 
 def cmc_five_column(filename):
   """cmc_four_column(filename) -> cmc_scores
-  
+
   Loads scores to compute CMC curves from a file in five column format.
   The four column file needs to be in the same format as described in :py:func:`five_column`,
   and the ``test_label`` (column 4) has to contain the test/probe file name or a probe id.
@@ -286,35 +269,19 @@ def cmc_five_column(filename):
   # read four column list
   for (client_id, _, probe_id, probe_name, score) in five_column(filename):
     # check in which dict we have to put the score
-    if client_id == probe_id:
-      correct_dict = pos_dict
-    else:
-      correct_dict = neg_dict
+    correct_dict = pos_dict if client_id == probe_id else neg_dict
+
     # append score
     if probe_name in correct_dict:
       correct_dict[probe_name].append(score)
     else:
       correct_dict[probe_name] = [score]
 
-  # convert to lists of tuples of ndarrays
-  retval = []
-  import logging
-  logger = logging.getLogger('bob')
-
-  for probe_name in sorted(pos_dict.keys()):
-    if probe_name in neg_dict:
-      retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
-    else:
-      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
-  # test if there are probes for which only negatives exist
-  for probe_name in sorted(neg_dict.keys()):
-    if not probe_name in pos_dict.keys():
-       logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
+  # convert that into the desired format
+  return _convert_cmc_scores(neg_dict, pos_dict)
 
-  return retval
 
-
-def load_score(filename, ncolumns=None):
+def load_score(filename, ncolumns = 4):
   """Load scores using numpy.loadtxt and return the data as a numpy array.
 
   **Parameters:**
@@ -333,11 +300,8 @@ def load_score(filename, ncolumns=None):
     'claimed_id', 'real_id', 'test_label', and ['model_label']
 
   """
-  if ncolumns is None:
-    ncolumns = 4
 
-  def convertfunc(x):
-    return x
+  convertfunc = lambda x : x
 
   if ncolumns == 4:
     names = ('claimed_id', 'real_id', 'test_label', 'score')
@@ -410,3 +374,13 @@ def dump_score(filename, score_lines):
   else:
     raise ValueError("Only scores with 4 and 5 columns are supported.")
   numpy.savetxt(filename, score_lines, fmt=fmt)
+
+def _convert_cmc_scores(neg_dict, pos_dict):
+  """Converts the negative and positive scores read with :py:func:`cmc_four_column` or :py:func:`cmc_four_column` into a format that is handled by the :py:func:`bob.measure.cmc` and similar functions."""
+  # convert to lists of tuples of ndarrays (or None)
+  probe_names = sorted(set(neg_dict.keys()).union(set(pos_dict.keys())))
+  # get all scores in the desired format
+  return [(
+    numpy.array(neg_dict[probe_name], numpy.float64) if probe_name in neg_dict else None,
+    numpy.array(pos_dict[probe_name], numpy.float64) if probe_name in pos_dict else None
+  ) for probe_name in probe_names]

bob/measure/plot.py

@@ -3,10 +3,34 @@
 # Chakka Murali Mohan, Trainee, IDIAP Research Institute, Switzerland.
 # Mon 23 May 2011 14:36:14 CEST
 
+def log_values(min_step = -4, counts_per_step = 4):
+  """log_values(min_step, counts_per_step) -> log_list
+
+  This function computes log-scaled values between :math:`10^{M}` and 1 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a negative integer.
+  The integral ``counts_per_step`` value defines how many values between two adjacent powers of 10 will be created.
+  The total number of values will be ``-min_step * counts_per_step + 1``.
+
+  **Parameters:**
+
+  ``min_step`` : int (negative)
+    The power of 10 that will be the minimum value. E.g., the default ``-4`` will result in the first number to be :math:`10^{-4}` = ``0.00001`` or ``0.01%``
+
+  ``counts_per_step`` : int (positive)
+    The number of values that will be put between two adjacent powers of 10.
+    With the default value ``4`` (and default values of ``min_step``), we will get ``log_list[0] == 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``.
+
+  **Returns**
+
+  ``log_list`` : [float]
+    A list of logarithmically scaled values between :math:`10^{M}` and 1.
+  """
+  import math
+  return [math.pow(10., i * 1./counts_per_step) for i in range(min_step*counts_per_step,0)] + [1.]
+
 """Methods to plot error analysis figures such as ROC, precision-recall curve, EPC and DET"""
 
 def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
-  """Plots Receiver Operating Charactaristic (ROC) curve.
+  """Plots Receiver Operating Characteristic (ROC) curve.
 
   This method will call ``matplotlib`` to plot the ROC curve for a system which
   contains a particular set of negatives (impostors) and positives (clients)
@@ -51,6 +75,46 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
     return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
 
 
+def roc_for_far(negatives, positives, far_values = log_values(), **kwargs):
+  """Plots Receiver Operating Characteristic (ROC) curve for the given list of False Acceptance Rates (FAR).
+
+  This method will call ``matplotlib`` to plot the ROC curve for a system which
+  contains a particular set of negatives (impostors) and positives (clients)
+  scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. All parameters
+  passed with exception of the three first parameters of this method will be
+  directly passed to the plot command.
+
+  The plot will represent the False Acceptance Rate (FAR) on the horizontal axis and the Correct Acceptance Rate (CAR) on the vertical axis.
+  The values for the axis will be computed using :py:func:`bob.measure.roc_for_far`.
+
+  .. note::
+
+    This function does not initiate and save the figure instance, it only
+    issues the plotting command. You are the responsible for setting up and
+    saving the figure as you see fit.
+
+  **Parameters:**
+
+  ``negatives, positives`` : array_like(1D, float)
+    The list of negative and positive scores forwarded to :py:func:`bob.measure.roc`
+
+  ``far_values`` : [float]
+    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
+
+  ``kwargs`` : keyword arguments
+    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
+
+  **Returns:**
+
+  The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.semilogx`.
+  """
+
+  from matplotlib import pyplot
+  from . import roc_for_far as calc
+  out = calc(negatives, positives, far_values)
+  return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
+
+
 def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
   """Plots Precision-Recall curve.
 
@@ -341,3 +405,59 @@ def cmc(cmc_scores, logx = True, **kwargs):
     pyplot.plot(range(1, len(out)+1), out * 100, **kwargs)
 
   return len(out)
+
+
+def detection_identification_curve(cmc_scores, far_values = log_values(), rank = 1, logx = True, **kwargs):
+  """Plots the Detection & Identification curve over the FAR for the given FAR values.
+  This curve is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
+  It requires to have at least one open set probe item, i.e., with no corresponding gallery, such that the positives for that pair are ``None``.
+
+  The detection and identification curve first computes FAR thresholds based on the out-of-set probe scores (negative scores).
+  For each probe item, the **maximum** negative score is used.
+  Then, it plots the detection and identification rates for those thresholds, which are based on the in-set probe scores only.
+  See [LiJain2005]_ for more details.
+
+  **Parameters:**
+
+  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
+    See :py:func:`bob.measure.detection_identification_rate`
+
+  ``far_values`` : [float]
+    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
+
+  ``rank`` : int or ``None``
+    The rank for which the curve should be plotted, 1 by default.
+
+  ``logx`` : bool
+    Plot the FAR axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``)
+
+  ``kwargs`` : keyword arguments
+    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.
+
+  **Returns:**
+
+  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
+
+  .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
+  """
+
+  import numpy
+  from matplotlib import pyplot
+  from . import far_threshold, detection_identification_rate
+
+  # for each probe, for which no positives exists, get the highest negative score; and sort them to compute the FAR thresholds
+  negatives = sorted(max(neg) for neg,pos in cmc_scores if (pos is None or not numpy.array(pos).size) and neg is not None)
+  if not negatives:
+    raise ValueError("There need to be at least one pair with only negative scores")
+
+  # compute thresholds based on FAR values
+  thresholds = [far_threshold(negatives, [], v, True) for v in far_values]
+
+  # compute detection and identification rate based on the thresholds for the given rank
+  rates = [100.*detection_identification_rate(cmc_scores, t, rank) for t in thresholds]
+
+  # plot curve
+  if logx:
+    return pyplot.semilogx(far_values, rates, **kwargs)
+  else:
+    return pyplot.plot(far_values, rates, **kwargs)

bob/measure/script/plot_cmc.py

@@ -30,6 +30,7 @@ def parse_command_line(command_line_options):
   parser.add_argument('-s', '--score-file', required = True, help = 'The score file in 4 or 5 column format to test.')
   parser.add_argument('-o', '--output-pdf-file', default = 'cmc.pdf', help = 'The PDF file to write.')