Commit 7b50ba95 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Evaluations

parent 1da1ff48
......@@ -52,18 +52,20 @@ def command_line_arguments(command_line_parameters):
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-d', '--dev-files', required=True, nargs='+', help = "A list of score files of the development set.")
parser.add_argument('-n', '--report-name', default="report", help = "The name of the report")
parser.add_argument('-r', '--roc', action='store_true', default=False, help="Add ROC in the report")
parser.add_argument('-e', '--det', action='store_true', default=False, help="Add DET in the report")
parser.add_argument('-s', '--directory', default = '.', help = "A directory, where to find the --dev-files and the --eval-files")
parser.add_argument('-c', '--criterion', choices = ('EER', 'HTER'), help = "If given, the threshold of the development set will be computed with this criterion.")
parser.add_argument('--cost', default=0.99, help='Cost for FAR in minDCF')
parser.add_argument('-r', '--rr', action = 'store_true', help = "If given, the Recognition Rate will be computed.")
parser.add_argument('-l', '--legends', nargs='+', help = "A list of legend strings used for ROC, CMC and DET plots; THE NUMBER OF PLOTS SHOULD BE MULTIPLE OF THE NUMBER OF LEGGENDS. IN THAT WAY, EACH SEGMENT WILL BE AVERAGED")
parser.add_argument('-i', '--linestyle', nargs='+', help = "A list of line styles ROC, CMC and DET plots; THE NUMBER OF PLOTS SHOULD BE MULTIPLE OF THE NUMBER OF LEGGENDS. IN THAT WAY, EACH SEGMENT WILL BE AVERAGED")
parser.add_argument('-F', '--legend-font-size', type=int, default=18, help = "Set the font size of the legends.")
parser.add_argument('-P', '--legend-position', type=int, help = "Set the font size of the legends.")
parser.add_argument('-R', '--roc', help = "If given, ROC curves will be plotted into the given pdf file.")
parser.add_argument('-D', '--det', help = "If given, DET curves will be plotted into the given pdf file.")
parser.add_argument('-C', '--cmc', help = "If given, CMC curves will be plotted into the given pdf file.")
parser.add_argument('--parser', default = '4column', choices = ('4column', '5column'), help="The style of the resulting score files. The default fits to the usual output of score files.")
# add verbose option
......@@ -89,7 +91,10 @@ def command_line_arguments(command_line_parameters):
return args
def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
def _plot_roc(scores_input, colors, labels, title, linestyle=None,fontsize=18, position=None):
linestyle = ['-','--','-','--','-','--']
if position is None: position = 4
figure = pyplot.figure()
......@@ -112,7 +117,11 @@ def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
frr_average = numpy.mean(frrs_accumulator, axis=0)
far_average = numpy.mean(fars_accumulator, axis=0); far_std = numpy.std(fars_accumulator, axis=0)
pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
if(linestyle is not None):
pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i], ls=linestyle[i].replace("\\",""))
else:
pyplot.semilogx(frr_average*100, 100. - 100.0*far_average, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
pyplot.errorbar(frr_average*100, 100. - 100.0*far_average, far_std*100, lw=0.5, ms=10)
offset += step
......@@ -135,6 +144,7 @@ def _plot_roc(scores_input, colors, labels, title, fontsize=18, position=None):
def _plot_det(scores_input, colors, labels, title, fontsize=18, position=None):
if position is None: position = 1
# open new page for current plot
figure = pyplot.figure(figsize=(8.2,8))
......@@ -179,6 +189,8 @@ def _plot_det(scores_input, colors, labels, title, fontsize=18, position=None):
return figure
def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
linestyle = ['-','--','-','--','-','--']
if position is None: position = 4
# open new page for current plot
figure = pyplot.figure()
......@@ -205,7 +217,8 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
cmc_std = numpy.std(cmc_accumulator, axis=0); cmc_std[-1]
cmc_average = numpy.mean(cmc_accumulator, axis=0)
pyplot.semilogx(range(1, cmc_average.shape[0]+1), cmc_average * 100, lw=2, ms=10, mew=1.5, label=labels[i])
pyplot.semilogx(range(1, cmc_average.shape[0]+1), cmc_average * 100, lw=2, ms=10, mew=1.5, label=labels[i], ls=linestyle[i])
pyplot.errorbar(range(1, cmc_average.shape[0]+1), cmc_average*100, cmc_std*100, lw=0.5, ms=10)
offset += step
......@@ -222,6 +235,15 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
return figure
#def generate_html(cmcs, path):
#"""
#Generates an
#"""
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
......@@ -231,72 +253,65 @@ def main(command_line_parameters=None):
cmap = pyplot.cm.get_cmap(name='hsv')
colors = [cmap(i) for i in numpy.linspace(0, 1.0, len(args.dev_files)+1)]
if args.criterion or args.roc or args.det:
#Creating a multipage PDF
pdf = PdfPages(args.report_name + ".pdf")
################ PLOTING CMC ##############
logger.info("Loading CMC data on the development ")
cmc_parser = {'4column' : bob.measure.load.cmc_four_column, '5column' : bob.measure.load.cmc_five_column}[args.parser]
cmcs_dev = [cmc_parser(f) for f in args.dev_files]
logger.info("Plotting CMC curves")
try:
# create a separate figure for dev and eval
pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CUHK-CUFS CMC between 5 splits", args.legend_font_size, args.legend_position))
except RuntimeError as e:
raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
#if args.rr:
#logger.info("Computing recognition rate on the development ")
#for i in range(len(cmcs_dev)):
#rr = bob.measure.recognition_rate(cmcs_dev[i])
#print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.))
################ PLOTING CMC ##############
if args.roc or args.det:
score_parser = {'4column' : bob.measure.load.split_four_column, '5column' : bob.measure.load.split_five_column}[args.parser]
# First, read the score files
logger.info("Loading %d score files of the development set", len(args.dev_files))
scores_dev = [score_parser(os.path.join(args.directory, f)) for f in args.dev_files]
if args.criterion:
logger.info("Computing %s on the development " % args.criterion )
for i in range(len(scores_dev)):
# compute threshold on development set
threshold = {'EER': bob.measure.eer_threshold, 'HTER' : bob.measure.min_hter_threshold} [args.criterion](scores_dev[i][0], scores_dev[i][1])
# apply threshold to development set
far, frr = bob.measure.farfrr(scores_dev[i][0], scores_dev[i][1], threshold)
print("The %s of the development set of '%s' is %2.3f%%" % (args.criterion, args.legends[i], (far + frr) * 50.)) # / 2 * 100%
scores_dev = [score_parser(f) for f in args.dev_files]
################ PLOTING ROC ##############
if args.roc:
logger.info("Plotting ROC curves to file '%s'", args.roc)
logger.info("Plotting ROC curves ")
try:
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.roc)
# create a separate figure for dev and eval
pdf.savefig(_plot_roc(scores_dev, colors, args.legends, "CUHK-CUFS ROC Curve between 5 splits", args.legend_font_size, args.legend_position))
#del frrs_dev
pdf.close()
except RuntimeError as e:
raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
################ PLOTING DET ##############
if args.det:
logger.info("Computing DET curves on the development ")
#dets_dev = [bob.measure.det(scores[0], scores[1], 1000) for scores in scores_dev]
logger.info("Plotting DET curves to file '%s'", args.det)
logger.info("Plotting DET curves")
try:
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.det)
# create a separate figure for dev and eval
pdf.savefig(_plot_det(scores_dev, colors, args.legends, "CUHK-CUFS DET between 5 splits", args.legend_font_size, args.legend_position))
#del dets_dev
pdf.close()
except RuntimeError as e:
raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
pdf.close()
if args.cmc or args.rr:
logger.info("Loading CMC data on the development ")
cmc_parser = {'4column' : bob.measure.load.cmc_four_column, '5column' : bob.measure.load.cmc_five_column}[args.parser]
cmcs_dev = [cmc_parser(os.path.join(args.directory, f)) for f in args.dev_files]
if args.cmc:
logger.info("Plotting CMC curves to file '%s'", args.cmc)
try:
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.cmc)
# create a separate figure for dev and eval
pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CUHK-CUFS CMC between 5 splits", args.legend_font_size, args.legend_position))
pdf.close()
except RuntimeError as e:
raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
if args.rr:
logger.info("Computing recognition rate on the development ")
for i in range(len(cmcs_dev)):
rr = bob.measure.recognition_rate(cmcs_dev[i])
print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# Thu 12 Nov 2015 16:35:08 CET
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""
This script will print a list of miss classification per identity
"""
import bob.io.base
import bob.io.image
import bob.measure
import argparse
import numpy, math
import os
import bob.db.cuhk_cufs
# enable LaTeX interpreter
import bob.core
logger = bob.core.log.setup("bob.bio.base")
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('score_file', help = "The file with the scores of the development set.")
parser.add_argument('preprocess_dir', help = "The directory with the pre-processed files.")
parser.add_argument('-o', '--output-dir', default="./temp", help = "The output directory.")
parser.add_argument('-e', '--extension', default=".hdf5", help = "The extension of the files.")
parser.add_argument('-p', '--protocol', default="search_split1_p2s", help = "Protocol to be executed")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
# set verbosity level
bob.core.log.set_verbosity_level(logger, args.verbose)
return args
def normalize4save(img):
return (255 * ((img - numpy.min(img)) / (numpy.max(img)-numpy.min(img)))).astype("uint8")
def get_scores_from_client(score_file, client_id):
"""
Organize the scores in genuine and the impostor
"""
output = {}
output['genuine'] = []
output['impostor'] = []
for client, probe, file_name, score in bob.measure.load.four_column(score_file):
if(int(client)==int(client_id)):
if(int(client)==int(probe)):
output['genuine'].append([file_name,score])
else:
output['impostor'].append([file_name,score])
return output
def generate_html_output(client_id, scores, img_dir, db, args):
"""
For each client, will compute the RR and return the pair of comparisons
"""
enroll_files = db.objects(protocol=args.protocol, groups="dev", purposes="enroll", model_ids=[client_id])
enroll_html = ""
for e in enroll_files:
img = normalize4save(bob.io.base.load(os.path.join(args.preprocess_dir, e.path)+args.extension))
img_file = os.path.join(img_dir,e.path)+".jpg"
bob.io.base.create_directories_safe(os.path.dirname(img_file))
bob.io.base.save(img, img_file)
enroll_html += "<img src='{0}'>".format(img_file.replace(args.output_dir,"./"))
genuine_html = "<tr><th colspan='3'><b>Client {0}</b></th></tr>\n".format(client_id)
max_genuine_score = -numpy.inf
for genuine in scores['genuine']:
img_file = os.path.join(args.preprocess_dir, genuine[0])+args.extension
img = normalize4save(bob.io.base.load(img_file))
img_file = os.path.join(img_dir,genuine[0])+".jpg"
bob.io.base.create_directories_safe(os.path.dirname(img_file))
bob.io.base.save(img, img_file)
genuine_html += "<tr><td>" + enroll_html + "</td> <td>" + "<img src='{0}'>".format(img_file.replace(args.output_dir,"./")) + "</td> <td style=\"color:blue;\"> <b>" + str(genuine[1]) + "</b></td> </tr>\n"
max_genuine_score = max(max_genuine_score, genuine[1])
impostor_html = ""
for impostor in scores['impostor']:
#Do only when the score of the impostor is bigger than the genuine score
if(impostor[1] > max_genuine_score):
img_file = os.path.join(args.preprocess_dir, impostor[0])+args.extension
img = normalize4save(bob.io.base.load(img_file))
img_file = os.path.join(img_dir,impostor[0])+".jpg"
bob.io.base.create_directories_safe(os.path.dirname(img_file))
bob.io.base.save(img, img_file)
impostor_html += "<tr><td>" + enroll_html + "</td> <td>" + "<img src='{0}'>".format(img_file.replace(args.output_dir,"./")) + "</td> <td style=\"color:red;\"><b>" + str(impostor[1]) + "</b></td> </tr>\n"
#Print the columns only if has false acceptance
if(impostor_html!=""):
return genuine_html + impostor_html
else:
return ""
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
db = bob.db.cuhk_cufs.Database(original_directory = args.preprocess_dir, original_extension=args.extension)
clients = db.clients(protocol=args.protocol, groups="dev")
img_dir = os.path.join(args.output_dir,"img")
bob.io.base.create_directories_safe(args.output_dir)
bob.io.base.create_directories_safe(img_dir)
html = "<html><body>\n"
html += " <style type=\"text/css\">\n"
html += "table, th, td {border: 1px solid black;}\n"
html += "table,p {font-size:16px;font-family: Arial, Helvetica, sans-serif;border-collapse: collapse; border-spacing: 0; width: 100%; }"
html += "td,th,p { border: 1px solid #ddd; text-align: left; padding: 8px;}"
html += "th { padding-top: 11px; padding-bottom: 11px; background-color: #4CAF50; color: white; text-align: center}"
html +="</style>\n"
html += " <p>Score file {0} </br> Protocol: {1} </br> Recognition rate: <b>{2}</b> </p>".format(args.score_file, args.protocol ,bob.measure.recognition_rate(bob.measure.load.cmc_four_column(args.score_file)))
html += " <table border='0'>\n"
i = 1
for c in clients:
print("Processing client {0} of {1}".format(i, len(clients)))
i = i + 1
scores = get_scores_from_client(args.score_file, c.id)
html += generate_html_output(c.id, scores, img_dir, db, args)
html += "</table>\n"
html += "</body></html>\n"
open(os.path.join(args.output_dir,"index.html"),'w').write(html)
......@@ -55,7 +55,8 @@ setup(
# scripts should be declared using this entry:
'console_scripts' : [
'evaluate_cufs.py = bob.db.cuhk_cufs.scripts.evaluate_cufs:main',
'evaluate_cufs.py = bob.db.cuhk_cufs.scripts.evaluate_cufs:main',
'html_report.py = bob.db.cuhk_cufs.scripts.html_report:main',
],
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment