From 4b5fa1c1e8d687f73d714f44f6be3df337a0b079 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Thu, 29 Sep 2016 11:35:44 +0200
Subject: [PATCH] Completely removes argparse requirement
---
bob/measure/script/apply_threshold.py | 9 ++
bob/measure/script/compute_perf.py | 4 +-
bob/measure/script/eval_threshold.py | 25 ++--
bob/measure/script/recurse_results.py | 194 ++++++++++++++------------
doc/guide.rst | 4 +-
5 files changed, 131 insertions(+), 105 deletions(-)
diff --git a/bob/measure/script/apply_threshold.py b/bob/measure/script/apply_threshold.py
index 36edcb6..dbf03a2 100644
--- a/bob/measure/script/apply_threshold.py
+++ b/bob/measure/script/apply_threshold.py
@@ -33,6 +33,11 @@ Examples:
import os
import sys
+import logging
+__logging_format__='[%(levelname)s] %(message)s'
+logging.basicConfig(format=__logging_format__)
+logger = logging.getLogger('bob')
+
from .eval_threshold import apthres
@@ -57,6 +62,10 @@ def main(user_input=None):
version=completions['version'],
)
+ # Sets-up logging
+ if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
+ elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
+
# handles threshold validation
try:
args['<threshold>'] = float(args['<threshold>'])
diff --git a/bob/measure/script/compute_perf.py b/bob/measure/script/compute_perf.py
index dcef7c1..a3e2b12 100644
--- a/bob/measure/script/compute_perf.py
+++ b/bob/measure/script/compute_perf.py
@@ -58,7 +58,7 @@ logger = logging.getLogger('bob')
def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
- """Prints a single output line that contains all info for a given criterium"""
+ """Prints a single output line that contains all info for a given criterion"""
from .. import eer_threshold, min_hter_threshold, farfrr
@@ -73,7 +73,7 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
test_far, test_frr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0
- print("[Min. criterium: %s] Threshold on Development set: %e" % (crit, thres))
+ print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
dev_ni = dev_neg.shape[0] #number of impostors
dev_fa = int(round(dev_far*dev_ni)) #number of false accepts
diff --git a/bob/measure/script/eval_threshold.py b/bob/measure/script/eval_threshold.py
index f3fc54a..d2a6a5d 100644
--- a/bob/measure/script/eval_threshold.py
+++ b/bob/measure/script/eval_threshold.py
@@ -19,11 +19,11 @@ Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
- -c <crit>, --criterium=<crit> The minimization criterium to use (choose
+ -c <crit>, --criterion=<crit> The minimization criterion to use (choose
between mhter, mwer or eer) [default: eer]
-w <float>, --cost=<float> The value w of the cost when minimizing using
the minimum weighter error rate (mwer)
- criterium. This value is ignored for eer or
+ criterion. This value is ignored for eer or
mhter criteria. [default: 0.5]
@@ -31,11 +31,11 @@ Examples:
1. Specify a different criteria (only mhter, mwer or eer accepted):
- $ %(prog)s --criterium=mhter scores.txt
+ $ %(prog)s --criterion=mhter scores.txt
2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4:
- $ %(prog)s --criterium=mwer --cost=0.4 scores.txt
+ $ %(prog)s --criterion=mwer --cost=0.4 scores.txt
3. Parse your input using a 5-column format
@@ -47,6 +47,11 @@ Examples:
import os
import sys
+import logging
+__logging_format__='[%(levelname)s] %(message)s'
+logging.basicConfig(format=__logging_format__)
+logger = logging.getLogger('bob')
+
def apthres(neg, pos, thres):
"""Prints a single output line that contains all info for the threshold"""
@@ -102,10 +107,14 @@ def main(user_input=None):
version=completions['version'],
)
- # validates criterium
+ # Sets-up logging
+ if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
+ elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
+
+ # validates criterion
valid_criteria = ('eer', 'mhter', 'mwer')
- if args['--criterium'] not in valid_criteria:
- raise docopt.DocoptExit("--criterium must be one of %s" % \
+ if args['--criterion'] not in valid_criteria:
+ raise docopt.DocoptExit("--criterion must be one of %s" % \
', '.join(valid_criteria))
# handles cost validation
@@ -121,7 +130,7 @@ def main(user_input=None):
from ..load import load_score, get_negatives_positives
neg, pos = get_negatives_positives(load_score(args['<scores>']))
- t = calculate(neg, pos, args['--criterium'], args['--cost'])
+ t = calculate(neg, pos, args['--criterion'], args['--cost'])
print("Threshold:", t)
apthres(neg, pos, t)
diff --git a/bob/measure/script/recurse_results.py b/bob/measure/script/recurse_results.py
index 4ff4065..0aa68c5 100644
--- a/bob/measure/script/recurse_results.py
+++ b/bob/measure/script/recurse_results.py
@@ -1,12 +1,9 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
-# Manuel Guenther <manuel.guenther@idiap.ch>
# Tue Jul 2 14:52:49 CEST 2013
-#
-# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+"""Collects results of verification experiments recursively, reports results
-"""
This script parses through the given directory, collects all results of
verification experiments that are stored in file with the given file name. It
supports the split into development and test set of the data, as well as
@@ -23,90 +20,62 @@ information are given in columns:
The measure type of the development set can be changed to compute "HTER" or
"FAR" thresholds instead, using the --criterion option.
+
+Usage: %(prog)s [-v...] [options]
+ %(prog)s --help
+ %(prog)s --version
+
+
+Options:
+ -h, --help Shows this help message and exit
+ -V, --version Prints the version and exits
+ -v, --verbose Increases the output verbosity level
+ -d <path>, --devel-name=<path> Name of the file containing the development
+ scores [default: scores-dev]
+ -e <path>, --eval-name=<path> Name of the file containing the evaluation
+ scores [default: scores-eval]
+ -D <dir>, --directory=<dir> The directory where the results should be
+ collected from [default: .]
+ -n <dir>, --nonorm-dir=<dir> Directory where the unnormalized scores are
+ found [default: nonorm]
+ -z <dir>, --ztnorm-dir=<dir> Directory where the normalized scores are
+ found [default: ztnorm]
+ -s, --sort If set, sorts the results
+ -k <key>, --sort-key=<key> Sorts the results according to the given key.
+ May be one of "nonorm_dev", "nonorm_eval",
+ "ztnorm_dev", "ztnorm_eval" or "dir"
+ [default: dir]
+ -c <crit>, --criterion=<crit> Report Equal Rates (EER) rather than Half
+ Total Error Rate (HTER). Choose between
+ "HTER", "EER" or "FAR" [default: HTER]
+ -o <path>, --output=<path> If set, outputs results to a file named after
+ the option. If not set, writes to the console
+
"""
import os
import sys
-#from apport.hookutils import default
-
-def get_args():
- """Parse the program options"""
-
- import argparse
- from .. import load
-
- # set up command line parser
- parser = argparse.ArgumentParser(description=__doc__,
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
- parser.add_argument('-d', '--devel-name', dest="dev", default="scores-dev", metavar="FILE",
- help = "Name of the file containing the development scores")
- parser.add_argument('-e', '--eval-name', dest="eval", default="scores-eval", metavar="FILE",
- help = "Name of the file containing the evaluation scores")
- parser.add_argument('-D', '--directory', default=".",
- help = "The directory where the results should be collected from.")
- parser.add_argument('-n', '--nonorm-dir', dest="nonorm", default="nonorm", metavar = "DIR",
- help = "Directory where the unnormalized scores are found")
- parser.add_argument('-z', '--ztnorm-dir', dest="ztnorm", default = "ztnorm", metavar = "DIR",
- help = "Directory where the normalized scores are found")
- parser.add_argument('-s', '--sort', dest="sort", action='store_true',
- help = "Sort the results")
- parser.add_argument('-k', '--sort-key', dest='key', default = 'dir', choices=['nonorm_dev','nonorm_eval','ztnorm_dev','ztnorm_eval','dir'],
- help = "Sort the results accordign to the given key")
- parser.add_argument('-c', '--criterion', dest='criterion', default = 'HTER', choices=['HTER', 'EER', 'FAR'],
- help = "Report Equal Rates (EER) rather than Half Total Error Rate (HTER)")
-
- parser.add_argument('-o', '--output', dest="output",
- help="Name of the output file that will contain the HTER scores")
-
- parser.add_argument('--self-test', action='store_true', help=argparse.SUPPRESS)
-
- parser.add_argument('-p', '--parser', dest="parser", default="4column", metavar="NAME.FUNCTION",
- help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details")
-
- # parse arguments
- args = parser.parse_args()
-
- # parse the score-parser
- if args.parser.lower() in ('4column', '4col'):
- args.parser = load.split_four_column
- elif args.parser.lower() in ('5column', '5col'):
- args.parser = load.split_five_column
- else: #try an import
- if args.parser.find('.') == -1:
- parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser)
-
- mod, fct = args.parser.rsplit('.', 2)
- import imp
- try:
- fp, pathname, description = imp.find_module(mod, ['.'] + sys.path)
- except Exception as e:
- parser.error("import error for '%s': %s" % (args.parser, e))
-
- try:
- pmod = imp.load_module(mod, fp, pathname, description)
- args.parser = getattr(pmod, fct)
- except Exception as e:
- parser.error("loading error for '%s': %s" % (args.parser, e))
- finally:
- fp.close()
-
- return args
+
+import logging
+__logging_format__='[%(levelname)s] %(message)s'
+logging.basicConfig(format=__logging_format__)
+logger = logging.getLogger('bob')
class Result:
def __init__(self, dir, args):
self.dir = dir
- self.m_args = args
+ self.args = args
self.nonorm_dev = None
self.nonorm_eval = None
self.ztnorm_dev = None
self.ztnorm_eval = None
def __calculate__(self, dev_file, eval_file = None):
+ from ..load import load_score, get_negatives_positives
from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr
- dev_neg, dev_pos = self.m_args.parser(dev_file)
+ dev_neg, dev_pos = get_negatives_positives(load_score(dev_file))
# switch which threshold function to use;
# THIS f***ing piece of code really is what python authors propose:
@@ -114,29 +83,31 @@ class Result:
'EER' : eer_threshold,
'HTER' : min_hter_threshold,
'FAR' : far_threshold
- } [self.m_args.criterion](dev_neg, dev_pos)
+ } [self.args['--criterion']](dev_neg, dev_pos)
# compute far and frr for the given threshold
dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold)
dev_hter = (dev_far + dev_frr)/2.0
if eval_file:
- eval_neg, eval_pos = self.m_args.parser(eval_file)
+ eval_neg, eval_pos = get_negatives_positives(load_score(eval_file))
eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold)
eval_hter = (eval_far + eval_frr)/2.0
else:
eval_hter = None
- if self.m_args.criterion == 'FAR':
+ if self.args['--criterion'] == 'FAR':
return (dev_frr, eval_frr)
else:
return (dev_hter, eval_hter)
def nonorm(self, dev_file, eval_file = None):
- (self.nonorm_dev, self.nonorm_eval) = self.__calculate__(dev_file, eval_file)
+ (self.nonorm_dev, self.nonorm_eval) = \
+ self.__calculate__(dev_file, eval_file)
def ztnorm(self, dev_file, eval_file = None):
- (self.ztnorm_dev, self.ztnorm_eval) = self.__calculate__(dev_file, eval_file)
+ (self.ztnorm_dev, self.ztnorm_eval) = \
+ self.__calculate__(dev_file, eval_file)
def __str__(self):
str = ""
@@ -153,12 +124,14 @@ class Result:
results = []
+
def add_results(args, nonorm, ztnorm = None):
+
r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args)
print("Adding results from directory", r.dir)
# check if the results files are there
- dev_file = os.path.join(nonorm, args.dev)
- eval_file = os.path.join(nonorm, args.eval)
+ dev_file = os.path.join(nonorm, args['--devel-name'])
+ eval_file = os.path.join(nonorm, args['--eval-name'])
if os.path.isfile(dev_file):
if os.path.isfile(eval_file):
r.nonorm(dev_file, eval_file)
@@ -166,8 +139,8 @@ def add_results(args, nonorm, ztnorm = None):
r.nonorm(dev_file)
if ztnorm:
- dev_file = os.path.join(ztnorm, args.dev)
- eval_file = os.path.join(ztnorm, args.eval)
+ dev_file = os.path.join(ztnorm, args['--devel-name'])
+ eval_file = os.path.join(ztnorm, args['--eval-name'])
if os.path.isfile(dev_file):
if os.path.isfile(eval_file):
r.ztnorm(dev_file, eval_file)
@@ -176,15 +149,17 @@ def add_results(args, nonorm, ztnorm = None):
results.append(r)
+
def recurse(args, path):
dir_list = os.listdir(path)
# check if the score directories are included in the current path
- if args.nonorm in dir_list:
- if args.ztnorm in dir_list:
- add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm))
+ if args['--nonorm-dir'] in dir_list:
+ if args['--ztnorm-dir'] in dir_list:
+ add_results(args, os.path.join(path, args['--nonorm-dir']),
+ os.path.join(path, args['--ztnorm-dir']))
else:
- add_results(args, os.path.join(path, args.nonorm))
+ add_results(args, os.path.join(path, args['--nonorm-dir']))
for e in dir_list:
real_path = os.path.join(path, e)
@@ -199,19 +174,52 @@ def table():
A += str(r) + "\n"
return A
-def main():
- args = get_args()
- recurse(args, args.directory)
+def main(user_input=None):
+
+ if user_input is not None:
+ argv = user_input
+ else:
+ argv = sys.argv[1:]
+
+ import docopt
+ import pkg_resources
+
+ completions = dict(
+ prog=os.path.basename(sys.argv[0]),
+ version=pkg_resources.require('bob.measure')[0].version
+ )
+
+ args = docopt.docopt(
+ __doc__ % completions,
+ argv=argv,
+ version=completions['version'],
+ )
+
+ # Sets-up logging
+ if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
+ elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
+
+ # checks sort-key
+ valid_sort_keys = 'nonorm_dev nonorm_eval ztnorm_dev ztnorm_eval dir'.split()
+ if args['--sort-key'] not in valid_sort_keys:
+ raise docopt.DocoptExit('--sort-key must be one of %s' % \
+ ', '.join(valid_sort_keys))
+
+ # checks criterion
+ valid_criterion = 'HTER EER FAR'.split()
+ if args['--criterion'] not in valid_criterion:
+ raise docopt.DocoptExit('--criterion must be one of %s' % \
+ ', '.join(valid_criterion))
+
+ recurse(args, args['--directory'])
- if args.sort:
+ if args['--sort']:
import operator
- results.sort(key=operator.attrgetter(args.key))
+ results.sort(key=operator.attrgetter(args['--sort-key']))
- if args.self_test:
- _ = table()
- elif args.output:
- f = open(args.output, "w")
+ if args['--output']:
+ f = open(args['--output'], "w")
f.writelines(table())
f.close()
else:
diff --git a/doc/guide.rst b/doc/guide.rst
index 288cf39..e99c90a 100644
--- a/doc/guide.rst
+++ b/doc/guide.rst
@@ -482,13 +482,13 @@ evaluation and plotting of development and test set data using our combined
.. code-block:: sh
$ bob_compute_perf.py --devel=development-scores-4col.txt --test=test-scores-4col.txt
- [Min. criterium: EER] Threshold on Development set: -4.787956e-03
+ [Min. criterion: EER] Threshold on Development set: -4.787956e-03
| Development | Test
-------+-----------------+------------------
FAR | 6.731% (35/520) | 2.500% (13/520)
FRR | 6.667% (26/390) | 6.154% (24/390)
HTER | 6.699% | 4.327%
- [Min. criterium: Min. HTER] Threshold on Development set: 3.411070e-03
+ [Min. criterion: Min. HTER] Threshold on Development set: 3.411070e-03
| Development | Test
-------+-----------------+------------------
FAR | 4.231% (22/520) | 1.923% (10/520)
--
GitLab