Evaluations

bob/db/cuhk_cufs/scripts/evaluate_cufs.py

@@ -52,18 +52,20 @@ def command_line_arguments(command_line_parameters):
       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
   parser.add_argument('-d', '--dev-files', required=True, nargs='+', help = "A list of score files of the development set.")
+  
+  parser.add_argument('-n', '--report-name', default="report", help = "The name of the report")
+  
+  parser.add_argument('-r', '--roc', action='store_true', default=False, help="Add ROC in the report")
+  parser.add_argument('-e', '--det', action='store_true', default=False, help="Add DET in the report")
 
-  parser.add_argument('-s', '--directory', default = '.', help = "A directory, where to find the --dev-files and the --eval-files")
 
-  parser.add_argument('-c', '--criterion', choices = ('EER', 'HTER'), help = "If given, the threshold of the development set will be computed with this criterion.")
-  parser.add_argument('--cost', default=0.99,  help='Cost for FAR in minDCF')
-  parser.add_argument('-r', '--rr', action = 'store_true', help = "If given, the Recognition Rate will be computed.")
   parser.add_argument('-l', '--legends', nargs='+', help = "A list of legend strings used for ROC, CMC and DET plots; THE NUMBER OF PLOTS SHOULD BE MULTIPLE OF THE NUMBER OF LEGGENDS. IN THAT WAY, EACH SEGMENT WILL BE AVERAGED")
+
+  parser.add_argument('-i', '--linestyle', nargs='+', help = "A list of line styles ROC, CMC and DET plots; THE NUMBER OF PLOTS SHOULD BE MULTIPLE OF THE NUMBER OF LEGGENDS. IN THAT WAY, EACH SEGMENT WILL BE AVERAGED")  
+  
+  
   parser.add_argument('-F', '--legend-font-size', type=int, default=18, help = "Set the font size of the legends.")
   parser.add_argument('-P', '--legend-position', type=int, help = "Set the font size of the legends.")
-  parser.add_argument('-R', '--roc', help = "If given, ROC curves will be plotted into the given pdf file.")
-  parser.add_argument('-D', '--det', help = "If given, DET curves will be plotted into the given pdf file.")
-  parser.add_argument('-C', '--cmc', help = "If given, CMC curves will be plotted into the given pdf file.")
   parser.add_argument('--parser', default = '4column', choices = ('4column', '5column'), help="The style of the resulting score files. The default fits to the usual output of score files.")
 
   # add verbose option
@@ -89,7 +91,10 @@ def command_line_arguments(command_line_parameters):
   return args
 
 
-def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
+def _plot_roc(scores_input, colors, labels, title, linestyle=None,fontsize=18, position=None):
+
+  linestyle = ['-','--','-','--','-','--']
+
   if position is None: position = 4
   figure = pyplot.figure()
     
@@ -112,7 +117,11 @@ def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
     frr_average = numpy.mean(frrs_accumulator, axis=0)
     far_average = numpy.mean(fars_accumulator, axis=0); far_std = numpy.std(fars_accumulator, axis=0)    
 
-    pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
+    if(linestyle is not None):
+      pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i], ls=linestyle[i].replace("\\",""))
+    else:
+      pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
+    
     pyplot.errorbar(frr_average*100, 100. - 100.0*far_average, far_std*100, lw=0.5, ms=10)    
     
     offset += step
@@ -135,6 +144,7 @@ def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
 
 
 def _plot_det(scores_input, colors, labels, title, fontsize=18, position=None):
+
   if position is None: position = 1
   # open new page for current plot
   figure = pyplot.figure(figsize=(8.2,8))
@@ -179,6 +189,8 @@ def _plot_det(scores_input, colors, labels, title, fontsize=18, position=None):
   return figure
 
 def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
+  linestyle = ['-','--','-','--','-','--']
+
   if position is None: position = 4
   # open new page for current plot
   figure = pyplot.figure()
@@ -205,7 +217,8 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
     cmc_std     = numpy.std(cmc_accumulator, axis=0); cmc_std[-1]
     cmc_average = numpy.mean(cmc_accumulator, axis=0)
     
-    pyplot.semilogx(range(1, cmc_average.shape[0]+1), cmc_average * 100, lw=2, ms=10, mew=1.5, label=labels[i])
+    pyplot.semilogx(range(1, cmc_average.shape[0]+1), cmc_average * 100, lw=2, ms=10, mew=1.5, label=labels[i], ls=linestyle[i])
+    
     pyplot.errorbar(range(1, cmc_average.shape[0]+1), cmc_average*100, cmc_std*100, lw=0.5, ms=10)
     offset += step    
 
@@ -222,6 +235,15 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
   return figure
 
 
+
+#def generate_html(cmcs, path):
+#"""
+#Generates an
+#"""
+
+
+
+
 def main(command_line_parameters=None):
   """Reads score files, computes error measures and plots curves."""
 
@@ -231,72 +253,65 @@ def main(command_line_parameters=None):
   cmap = pyplot.cm.get_cmap(name='hsv')
   colors = [cmap(i) for i in numpy.linspace(0, 1.0, len(args.dev_files)+1)]
 
-  if args.criterion or args.roc or args.det:
+  #Creating a multipage PDF
+  pdf = PdfPages(args.report_name + ".pdf")
+
+ 
+  ################ PLOTING CMC ##############
+  logger.info("Loading CMC data on the development ")
+  cmc_parser = {'4column' : bob.measure.load.cmc_four_column, '5column' : bob.measure.load.cmc_five_column}[args.parser]
+  cmcs_dev = [cmc_parser(f) for f in args.dev_files]
+  logger.info("Plotting CMC curves")
+  try:
+    # create a separate figure for dev and eval
+    pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CUHK-CUFS CMC between 5 splits", args.legend_font_size, args.legend_position))
+  except RuntimeError as e:
+    raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
+    
+    
+  #if args.rr:
+    #logger.info("Computing recognition rate on the development ")
+    #for i in range(len(cmcs_dev)):
+      #rr = bob.measure.recognition_rate(cmcs_dev[i])
+      #print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.))
+
+ 
+  ################ PLOTING CMC ##############
+  if args.roc or args.det:
     score_parser = {'4column' : bob.measure.load.split_four_column, '5column' : bob.measure.load.split_five_column}[args.parser]
 
     # First, read the score files
     logger.info("Loading %d score files of the development set", len(args.dev_files))
-    scores_dev = [score_parser(os.path.join(args.directory, f)) for f in args.dev_files]
-
-
-    if args.criterion:
-      logger.info("Computing %s on the development " % args.criterion )
-      for i in range(len(scores_dev)):
-        # compute threshold on development set
-        threshold = {'EER': bob.measure.eer_threshold, 'HTER' : bob.measure.min_hter_threshold} [args.criterion](scores_dev[i][0], scores_dev[i][1])
-        # apply threshold to development set
-        far, frr = bob.measure.farfrr(scores_dev[i][0], scores_dev[i][1], threshold)
-        print("The %s of the development set of '%s' is %2.3f%%" % (args.criterion, args.legends[i], (far + frr) * 50.)) # / 2 * 100%
-
+    scores_dev = [score_parser(f) for f in args.dev_files]
 
 
+    ################ PLOTING ROC ##############
     if args.roc:
-      logger.info("Plotting ROC curves to file '%s'", args.roc)
+      logger.info("Plotting ROC curves ")
       try:
-        # create a multi-page PDF for the ROC curve
-        pdf = PdfPages(args.roc)
         # create a separate figure for dev and eval
         pdf.savefig(_plot_roc(scores_dev, colors, args.legends, "CUHK-CUFS ROC Curve between 5 splits", args.legend_font_size, args.legend_position))
         #del frrs_dev
-        pdf.close()
       except RuntimeError as e:
         raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
 
+
+    ################ PLOTING DET ##############
     if args.det:
       logger.info("Computing DET curves on the development ")
       #dets_dev = [bob.measure.det(scores[0], scores[1], 1000) for scores in scores_dev]
 
-      logger.info("Plotting DET curves to file '%s'", args.det)
+      logger.info("Plotting DET curves")
       try:
-        # create a multi-page PDF for the ROC curve
-        pdf = PdfPages(args.det)
         # create a separate figure for dev and eval
         pdf.savefig(_plot_det(scores_dev, colors, args.legends, "CUHK-CUFS DET between 5 splits", args.legend_font_size, args.legend_position))
         #del dets_dev
-        pdf.close()
       except RuntimeError as e:
         raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
 
+  pdf.close()
+  
+  
+  
 
-  if args.cmc or args.rr:
-    logger.info("Loading CMC data on the development ")
-    cmc_parser = {'4column' : bob.measure.load.cmc_four_column, '5column' : bob.measure.load.cmc_five_column}[args.parser]
-    cmcs_dev = [cmc_parser(os.path.join(args.directory, f)) for f in args.dev_files]
-
-  if args.cmc:
-    logger.info("Plotting CMC curves to file '%s'", args.cmc)
-    try:
-      # create a multi-page PDF for the ROC curve
-      pdf = PdfPages(args.cmc)
-      # create a separate figure for dev and eval
-      pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CUHK-CUFS CMC between 5 splits", args.legend_font_size, args.legend_position))
-      pdf.close()
-    except RuntimeError as e:
-      raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
-
-  if args.rr:
-    logger.info("Computing recognition rate on the development ")
-    for i in range(len(cmcs_dev)):
-      rr = bob.measure.recognition_rate(cmcs_dev[i])
-      print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.))
 

bob/db/cuhk_cufs/scripts/html_report.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# Thu 12 Nov 2015 16:35:08 CET 
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+"""
+This script will print a list of miss classification per identity
+"""
+
+import bob.io.base
+import bob.io.image
+import bob.measure
+
+import argparse
+import numpy, math
+import os
+import bob.db.cuhk_cufs
+
+# enable LaTeX interpreter
+
+import bob.core
+logger = bob.core.log.setup("bob.bio.base")
+
+
+
+def command_line_arguments(command_line_parameters):
+  """Parse the program options"""
+
+  # set up command line parser
+  parser = argparse.ArgumentParser(description=__doc__,
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+  parser.add_argument('score_file',  help = "The file with the scores of the development set.")
+  
+  parser.add_argument('preprocess_dir', help = "The directory with the pre-processed files.")  
+  
+  parser.add_argument('-o', '--output-dir', default="./temp", help = "The output directory.")
+
+  parser.add_argument('-e', '--extension', default=".hdf5", help = "The extension of the files.")
+
+  parser.add_argument('-p', '--protocol', default="search_split1_p2s", help = "Protocol to be executed")
+
+  # add verbose option
+  bob.core.log.add_command_line_option(parser)
+
+  # parse arguments
+  args = parser.parse_args(command_line_parameters)
+
+  # set verbosity level
+  bob.core.log.set_verbosity_level(logger, args.verbose)
+
+  return args
+
+
+
+def normalize4save(img):
+  return (255 * ((img - numpy.min(img)) / (numpy.max(img)-numpy.min(img)))).astype("uint8")
+
+
+
+def get_scores_from_client(score_file, client_id):
+  """
+  Organize the scores in genuine and the impostor 
+  """
+
+  output = {}
+  output['genuine'] = []
+  output['impostor']  =  []
+
+  for client, probe, file_name, score in bob.measure.load.four_column(score_file):
+    
+    if(int(client)==int(client_id)):
+
+      if(int(client)==int(probe)):
+        output['genuine'].append([file_name,score])
+      else:
+        output['impostor'].append([file_name,score])
+        
+  return output
+  
+
+
+def generate_html_output(client_id, scores, img_dir, db, args):
+  """
+  For each client, will compute the RR and return the pair of comparisons
+  """  
+
+  enroll_files = db.objects(protocol=args.protocol, groups="dev", purposes="enroll", model_ids=[client_id])
+
+  enroll_html = ""
+  for e in enroll_files:
+    img      = normalize4save(bob.io.base.load(os.path.join(args.preprocess_dir, e.path)+args.extension))
+    img_file = os.path.join(img_dir,e.path)+".jpg"
+    bob.io.base.create_directories_safe(os.path.dirname(img_file))
+    bob.io.base.save(img, img_file)
+    enroll_html += "<img src='{0}'>".format(img_file.replace(args.output_dir,"./"))
+
+  genuine_html = "<tr><th colspan='3'><b>Client {0}</b></th></tr>\n".format(client_id)
+  max_genuine_score = -numpy.inf
+  for genuine in scores['genuine']:
+    img_file = os.path.join(args.preprocess_dir, genuine[0])+args.extension
+    img = normalize4save(bob.io.base.load(img_file))
+    img_file = os.path.join(img_dir,genuine[0])+".jpg"
+    
+    bob.io.base.create_directories_safe(os.path.dirname(img_file))      
+    bob.io.base.save(img, img_file)
+    genuine_html += "<tr><td>" + enroll_html + "</td> <td>" + "<img src='{0}'>".format(img_file.replace(args.output_dir,"./")) + "</td> <td style=\"color:blue;\"> <b>" + str(genuine[1]) + "</b></td> </tr>\n"
+    max_genuine_score = max(max_genuine_score, genuine[1])
+  
+  
+  impostor_html = ""
+  for impostor in scores['impostor']:  
+    #Do only when the score of the impostor is bigger than the genuine score
+    if(impostor[1] > max_genuine_score):
+      img_file = os.path.join(args.preprocess_dir, impostor[0])+args.extension
+      img = normalize4save(bob.io.base.load(img_file))
+      img_file = os.path.join(img_dir,impostor[0])+".jpg"
+      bob.io.base.create_directories_safe(os.path.dirname(img_file))
+      bob.io.base.save(img, img_file)
+      impostor_html += "<tr><td>" + enroll_html + "</td> <td>" + "<img src='{0}'>".format(img_file.replace(args.output_dir,"./")) + "</td> <td style=\"color:red;\"><b>" + str(impostor[1]) + "</b></td> </tr>\n"
+      
+    #Print the columns only if has false acceptance
+  if(impostor_html!=""):
+    return genuine_html + impostor_html
+  else:
+    return ""
+
+  
+
+def main(command_line_parameters=None):
+  """Reads score files, computes error measures and plots curves."""
+
+  args = command_line_arguments(command_line_parameters)
+
+  db = bob.db.cuhk_cufs.Database(original_directory = args.preprocess_dir, original_extension=args.extension)
+  clients = db.clients(protocol=args.protocol, groups="dev")
+
+  img_dir = os.path.join(args.output_dir,"img")
+  bob.io.base.create_directories_safe(args.output_dir)
+  bob.io.base.create_directories_safe(img_dir)
+
+
+  html = "<html><body>\n"
+  html += " <style type=\"text/css\">\n"
+  html += "table, th, td {border: 1px solid black;}\n" 
+  html += "table,p {font-size:16px;font-family: Arial, Helvetica, sans-serif;border-collapse: collapse; border-spacing: 0; width: 100%; }"
+  html += "td,th,p { border: 1px solid #ddd; text-align: left; padding: 8px;}"
+  html += "th { padding-top: 11px; padding-bottom: 11px; background-color: #4CAF50; color: white; text-align: center}"
+  html +="</style>\n"
+
+  html += " <p>Score file {0} </br> Protocol: {1} </br> Recognition rate: <b>{2}</b> </p>".format(args.score_file, args.protocol ,bob.measure.recognition_rate(bob.measure.load.cmc_four_column(args.score_file)))
+  html += " <table border='0'>\n"
+  i = 1
+  for c in clients:
+    print("Processing client {0} of {1}".format(i, len(clients)))
+    i = i + 1
+    scores = get_scores_from_client(args.score_file, c.id)
+    html += generate_html_output(c.id, scores, img_dir, db, args)
+      
+  html += "</table>\n"
+  html += "</body></html>\n"
+
+  open(os.path.join(args.output_dir,"index.html"),'w').write(html)
+  
+  
+    
+    
+
+

setup.py

@@ -55,7 +55,8 @@ setup(
       
       # scripts should be declared using this entry:
       'console_scripts' : [
-        'evaluate_cufs.py     = bob.db.cuhk_cufs.scripts.evaluate_cufs:main',
+        'evaluate_cufs.py = bob.db.cuhk_cufs.scripts.evaluate_cufs:main',
+        'html_report.py   = bob.db.cuhk_cufs.scripts.html_report:main',
       ],      
       
     },