Solved issue #6

bob/measure/__init__.py

@@ -130,22 +130,11 @@ def recognition_rate(cmc_scores, threshold=None):
         correct += 1.
         
     else:
-
       #If threshold is NOT None, we have an openset identification
-      if(len(pos)>0):
-        # if we have positive scores the comparison is considered correct
-        # if the positive score is higher than the threshold AND all negative scores
-        max_pos = numpy.max(pos)
-
-        if((threshold < max_pos) and (neg < max_pos).all()):
-            correct += 1.
-
-      else:
-        #If we don't have a positive score we only will consider 
-        #a correct classification if ALL the negative scores are smaller than the threshold
-        if (neg < threshold).all():
-          correct += 1.
+      max_pos = numpy.max(pos)
 
+      if((threshold < max_pos) and (neg < max_pos).all()):
+          correct += 1.
   # return relative number of correctly matched scores
   return correct / float(len(cmc_scores))
 

bob/measure/load.py

@@ -128,7 +128,7 @@ def split_four_column(filename):
 
   return (numpy.array(neg, numpy.float64), numpy.array(pos, numpy.float64))
 
-def cmc_four_column(filename, load_only_negatives=False):
+def cmc_four_column(filename):
   """
   cmc_four_column(filename) -> cmc_scores
   
@@ -148,9 +148,6 @@ def cmc_four_column(filename, load_only_negatives=False):
   ``filename`` : str or file-like
     The file that will be opened with :py:func:`open_file` containing the scores.
 
-  ``load_only_negatives`` : boolean
-    Set this argument to **True** if you want also to load the probes that has negative scores **only** (used for open-set recognition).
-
 
   **Returns:**
 
@@ -182,25 +179,17 @@ def cmc_four_column(filename, load_only_negatives=False):
   retval = []
   import logging
   logger = logging.getLogger('bob')
-  if(not load_only_negatives):
-    for probe_name in sorted(pos_dict.keys()):
-      if probe_name in neg_dict:
-        retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
-      else:
-        logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
-
-    #test if there are probes for which only negatives exist
-    for probe_name in sorted(neg_dict.keys()):
-      if not probe_name in pos_dict.keys():
-        logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
-
-  else:
-    for probe_name in sorted(pos_dict.keys()):
+  for probe_name in sorted(pos_dict.keys()):
+    if probe_name in neg_dict:
       retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
+    else:
+      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
+
+  #test if there are probes for which only negatives exist
+  for probe_name in sorted(neg_dict.keys()):
+    if not probe_name in pos_dict.keys():
+      logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
 
-    for probe_name in sorted(neg_dict.keys()):
-      if not probe_name in pos_dict.keys():
-        retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array([], numpy.float64)))
 
   return retval
   
@@ -288,7 +277,7 @@ def split_five_column(filename):
   return (numpy.array(neg, numpy.float64), numpy.array(pos, numpy.float64))
 
 
-def cmc_five_column(filename, load_only_negatives=False):
+def cmc_five_column(filename):
   """
   cmc_four_column(filename) -> cmc_scores
   
@@ -306,10 +295,6 @@ def cmc_five_column(filename, load_only_negatives=False):
   ``filename`` : str or file-like
     The file that will be opened with :py:func:`open_file` containing the scores.
 
-  ``load_only_negatives`` : boolean
-    Set this argument to **True** if you want also to load the probes that has negative scores **only** (used for open-set recognition).
-
-
   **Returns:**
 
   ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
@@ -336,25 +321,15 @@ def cmc_five_column(filename, load_only_negatives=False):
   retval = []
   import logging
   logger = logging.getLogger('bob')
-  if(not load_only_negatives):
 
-    for probe_name in sorted(pos_dict.keys()):
-      if probe_name in neg_dict:
-        retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
-      else:
-        logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
-    # test if there are probes for which only negatives exist
-    for probe_name in sorted(neg_dict.keys()):
-      if not probe_name in pos_dict.keys():
-         logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
-  else:
-  
-    for probe_name in sorted(pos_dict.keys()):
+  for probe_name in sorted(pos_dict.keys()):
+    if probe_name in neg_dict:
       retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
-
-    for probe_name in sorted(neg_dict.keys()):
-      if not probe_name in pos_dict.keys():
-        retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array([], numpy.float64)))
-  
+    else:
+      logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
+  # test if there are probes for which only negatives exist
+  for probe_name in sorted(neg_dict.keys()):
+    if not probe_name in pos_dict.keys():
+       logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
 
   return retval

bob/measure/test_error.py

@@ -326,20 +326,30 @@ def test_calibration():
 
 def test_open_set_recognition_rate():
   
+  far_value = 0.01
+  
   #No error files
-  scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"),   load_only_negatives=True)
-  assert bob.measure.recognition_rate(scores, threshold=0.5), 1.0
-  assert bob.measure.recognition_rate(scores, threshold=10.), 0.222222222222
+  cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"))
+  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set.txt"))
+  assert bob.measure.recognition_rate(cmc_scores), 1.0
+  assert bob.measure.recognition_rate(cmc_scores, threshold=0.5), 1.0    
+  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
+  assert bob.measure.recognition_rate(cmc_scores, threshold=t), 1.0
   
   #One error
-  scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-one-error.txt"), 
-  load_only_negatives=True)
-  assert bob.measure.recognition_rate(scores, threshold=0.5), 0.888888888889
-  assert bob.measure.recognition_rate(scores, threshold=10.), 0.222222222222
+  cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"))
+  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set.txt"))
+  assert bob.measure.recognition_rate(cmc_scores), 0.857142857143
+  assert bob.measure.recognition_rate(cmc_scores, threshold=0.5), 0.857142857143
+  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)  
+  assert bob.measure.recognition_rate(cmc_scores, threshold=t), 0.857142857143
 
   #Two errors
-  scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-two-errors.txt"), 
-  load_only_negatives=True)
-  assert bob.measure.recognition_rate(scores, threshold=0.5), 0.777777777778
-  assert bob.measure.recognition_rate(scores, threshold=10.), 0.111111111111  
+  cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"))
+  normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set.txt"))
+  assert bob.measure.recognition_rate(cmc_scores), 0.857142857143
+  assert bob.measure.recognition_rate(cmc_scores, threshold=0.5), 0.857142857143
+  t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)  
+  assert bob.measure.recognition_rate(cmc_scores, threshold=t), 0.0
+