From abf0e758f1d32c3fbf40cf4986f5686b56f22b55 Mon Sep 17 00:00:00 2001
From: Manuel Guenther <manuel.guenther@idiap.ch>
Date: Fri, 19 Jun 2015 16:17:19 +0200
Subject: [PATCH] Added collect_results script
---
bob/bio/base/script/collect_results.py | 217 +++++++++++++++++++++++++
bob/bio/base/test/test_scripts.py | 16 +-
doc/conf.py | 10 +-
setup.py | 1 +
4 files changed, 230 insertions(+), 14 deletions(-)
create mode 100644 bob/bio/base/script/collect_results.py
diff --git a/bob/bio/base/script/collect_results.py b/bob/bio/base/script/collect_results.py
new file mode 100644
index 00000000..b9438c0e
--- /dev/null
+++ b/bob/bio/base/script/collect_results.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Manuel Guenther <manuel.guenther@idiap.ch>
+# Tue Jul 2 14:52:49 CEST 2013
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
+
+"""
+This script parses through the given directory, collects all results of
+verification experiments that are stored in file with the given file name.
+It supports the split into development and test set of the data, as well as
+ZT-normalized scores.
+
+All result files are parsed and evaluated. For each directory, the following
+information are given in columns:
+
+ * The Equal Error Rate of the development set
+ * The Equal Error Rate of the development set after ZT-Normalization
+ * The Half Total Error Rate of the evaluation set
+ * The Half Total Error Rate of the evaluation set after ZT-Normalization
+ * The sub-directory where the scores can be found
+
+The measure type of the development set can be changed to compute "HTER" or
+"FAR" thresholds instead, using the --criterion option.
+"""
+
+
+import sys, os, glob
+import argparse
+
+import bob.measure
+import bob.core
+logger = bob.core.log.setup("bob.bio.base")
+
+def command_line_arguments(command_line_parameters):
+ """Parse the program options"""
+
+ # set up command line parser
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+ parser.add_argument('-d', '--devel-name', dest="dev", default="scores-dev", help = "Name of the file containing the development scores")
+ parser.add_argument('-e', '--eval-name', dest="eval", default="scores-eval", help = "Name of the file containing the evaluation scores")
+ parser.add_argument('-D', '--directory', default=".", help = "The directory where the results should be collected from; might include search patterns as '*'.")
+ parser.add_argument('-n', '--nonorm-dir', dest="nonorm", default="nonorm", help = "Directory where the unnormalized scores are found")
+ parser.add_argument('-z', '--ztnorm-dir', dest="ztnorm", default = "ztnorm", help = "Directory where the normalized scores are found")
+ parser.add_argument('-s', '--sort', action='store_true', help = "Sort the results")
+ parser.add_argument('-k', '--sort-key', dest='key', default = 'nonorm-dev', choices= ('nonorm-dev','nonorm-eval','ztnorm-dev','ztnorm-eval','dir'),
+ help = "Sort the results according to the given key")
+ parser.add_argument('-c', '--criterion', dest='criterion', default = 'EER', choices = ('EER', 'HTER', 'FAR'),
+ help = "Minimize the threshold on the development set according to the given criterion")
+
+ parser.add_argument('-o', '--output', help = "Name of the output file that will contain the EER/HTER scores")
+ parser.add_argument('--parser', default = '4column', choices = ('4column', '5column'), help="The style of the resulting score files; rarely changed")
+
+ parser.add_argument('--self-test', action='store_true', help=argparse.SUPPRESS)
+
+ bob.core.log.add_command_line_option(parser)
+
+ # parse arguments
+ args = parser.parse_args(command_line_parameters)
+
+ bob.core.log.set_verbosity_level(logger, args.verbose)
+
+ # assign the score file parser
+ args.parser = {'4column' : bob.measure.load.split_four_column, '5column' : bob.measure.load.split_five_column}[args.parser]
+
+ return args
+
+class Result:
+ """Class for collecting the results of one experiment."""
+ def __init__(self, dir, args):
+ self.dir = dir
+ self.m_args = args
+ self.nonorm_dev = None
+ self.nonorm_eval = None
+ self.ztnorm_dev = None
+ self.ztnorm_eval = None
+
+ def _calculate(self, dev_file, eval_file = None):
+ """Calculates the EER and HTER or FRR based on the threshold criterion."""
+ dev_neg, dev_pos = self.m_args.parser(dev_file)
+
+ # switch which threshold function to use;
+ # THIS f***ing piece of code really is what python authors propose:
+ threshold = {
+ 'EER' : bob.measure.eer_threshold,
+ 'HTER' : bob.measure.min_hter_threshold,
+ 'FAR' : bob.measure.far_threshold
+ } [self.m_args.criterion](dev_neg, dev_pos)
+
+ # compute far and frr for the given threshold
+ dev_far, dev_frr = bob.measure.farfrr(dev_neg, dev_pos, threshold)
+ dev_hter = (dev_far + dev_frr)/2.0
+
+ if eval_file:
+ eval_neg, eval_pos = self.m_args.parser(eval_file)
+ eval_far, eval_frr = bob.measure.farfrr(eval_neg, eval_pos, threshold)
+ eval_hter = (eval_far + eval_frr)/2.0
+ else:
+ eval_hter = None
+ eval_frr = None
+
+ if self.m_args.criterion == 'FAR':
+ return (dev_frr, eval_frr)
+ else:
+ return (dev_hter, eval_hter)
+
+ def nonorm(self, dev_file, eval_file = None):
+ self.nonorm_dev, self.nonorm_eval = self._calculate(dev_file, eval_file)
+
+ def ztnorm(self, dev_file, eval_file = None):
+ self.ztnorm_dev, self.ztnorm_eval = self._calculate(dev_file, eval_file)
+
+ def __str__(self):
+ str = ""
+ for v in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]:
+ if v:
+ val = "% 2.3f%%"%(v*100)
+ else:
+ val = "None"
+ cnt = 16-len(val)
+ str += " "*cnt + val
+ str += " %s"%self.dir
+ return str[5:]
+
+
+results = []
+
+def add_results(args, nonorm, ztnorm = None):
+ """Adds results of the given nonorm and ztnorm directories."""
+ r = Result(os.path.dirname(nonorm).replace(args.directory+"/", ""), args)
+ logger.info("Adding results from directory '%s'", r.dir)
+
+ # check if the results files are there
+ dev_file = os.path.join(nonorm, args.dev)
+ eval_file = os.path.join(nonorm, args.eval)
+ if os.path.isfile(dev_file):
+ if os.path.isfile(eval_file):
+ r.nonorm(dev_file, eval_file)
+ else:
+ r.nonorm(dev_file)
+
+ if ztnorm:
+ dev_file = os.path.join(ztnorm, args.dev)
+ eval_file = os.path.join(ztnorm, args.eval)
+ if os.path.isfile(dev_file):
+ if os.path.isfile(eval_file):
+ r.ztnorm(dev_file, eval_file)
+ else:
+ r.ztnorm(dev_file)
+
+ results.append(r)
+
+
+def recurse(args, path):
+ """Recurse the directory structure and collect all results that are stored in the desired file names."""
+ dir_list = os.listdir(path)
+
+ # check if the score directories are included in the current path
+ if args.nonorm in dir_list:
+ if args.ztnorm in dir_list:
+ add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm))
+ else:
+ add_results(args, os.path.join(path, args.nonorm))
+
+ for e in dir_list:
+ real_path = os.path.join(path, e)
+ if os.path.isdir(real_path):
+ recurse(args, real_path)
+
+
+def table():
+ """Generates a table containing all results in a nice format."""
+ A = " "*2 + 'dev nonorm'+ " "*5 + 'dev ztnorm' + " "*6 + 'eval nonorm' + " "*4 + 'eval ztnorm' + " "*12 + 'directory\n'
+ A += "-"*100+"\n"
+ for r in results:
+ A += str(r) + "\n"
+ return A
+
+
+def main(command_line_parameters = None):
+ """Iterates through the desired directory and collects all result files."""
+ args = command_line_arguments(command_line_parameters)
+
+ # collect results
+ directories = glob.glob(args.directory)
+ for directory in directories:
+ recurse(args, directory)
+
+ # sort results if desired
+ if args.sort:
+ import operator
+ results.sort(key=operator.attrgetter(args.key.replace('-','_')))
+
+ # print the results
+ if args.self_test:
+ table()
+ elif args.output:
+ f = open(args.output, "w")
+ f.writelines(table())
+ f.close()
+ else:
+ print (table())
diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py
index 7d40bc34..94c13f6e 100644
--- a/bob/bio/base/test/test_scripts.py
+++ b/bob/bio/base/test/test_scripts.py
@@ -278,18 +278,16 @@ def test_evaluate():
os.rmdir(test_dir)
-
-
-
-"""
-def test16_collect_results(self):
+def test_collect_results():
# simply test that the collect_results script works
test_dir = tempfile.mkdtemp(prefix='bobtest_')
- from facereclib.script.collect_results import main
- main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test'])
- os.rmdir(test_dir)
+ try:
+ from facereclib.script.collect_results import main
+ main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test'])
+ finally:
+ if os.path.exists(test_dir):
+ os.rmdir(test_dir)
-"""
@utils.grid_available
def test_grid_search():
diff --git a/doc/conf.py b/doc/conf.py
index c667df32..d102e79c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -58,7 +58,7 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
-project = u'Bob Example Project'
+project = u'Bobs interface for running biometric recognition experiments'
import time
copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
@@ -187,7 +187,7 @@ html_favicon = ''
#html_file_suffix = None
# Output file base name for HTML help builder.
-htmlhelp_basename = 'bob_example_project_doc'
+htmlhelp_basename = 'bob_bio_base_doc'
# -- Options for LaTeX output --------------------------------------------------
@@ -201,7 +201,7 @@ latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
- ('index', 'bob_example_project.tex', u'Bob',
+ ('index', 'bob_bio_base.tex', u'Bob',
u'Biometrics Group, Idiap Research Institute', 'manual'),
]
@@ -236,7 +236,7 @@ rst_epilog = ''
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- ('index', 'bob.example.project', u'Bob Example Project Documentation', [u'Idiap Research Institute'], 1)
+ ('index', 'bob.bio.base', u'Base tools to run biometric recognition experiments', [u'Idiap Research Institute'], 1)
]
# Default processing flags for sphinx
@@ -246,7 +246,7 @@ autodoc_default_flags = ['members', 'undoc-members', 'inherited-members', 'show-
# For inter-documentation mapping:
from bob.extension.utils import link_documentation
-intersphinx_mapping = link_documentation(['python', 'numpy', 'bob.io.base', 'bob.db.verification.utils'])
+intersphinx_mapping = link_documentation(['python', 'numpy', 'bob.io.base', 'bob.db.verification.utils', 'bob.bio.face', 'bob.bio.speaker', 'bob.bio.gmm', 'bob.bio.video', 'bob.bio.csu'])
def setup(app):
diff --git a/setup.py b/setup.py
index 3746ff5e..6811620e 100644
--- a/setup.py
+++ b/setup.py
@@ -106,6 +106,7 @@ setup(
'verify.py = bob.bio.base.script.verify:main',
'resources.py = bob.bio.base.script.resources:main',
'evaluate.py = bob.bio.base.script.evaluate:main',
+ 'collect_results.py = bob.bio.base.script.collect_results:main',
'grid_search.py = bob.bio.base.script.grid_search:main',
'preprocess.py = bob.bio.base.script.preprocess:main',
'extract.py = bob.bio.base.script.extract:main',
--
GitLab