Skip to content
Snippets Groups Projects
Commit 4b5fa1c1 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

Completely removes argparse requirement

parent c51614d9
No related branches found
No related tags found
1 merge request!15Docfix
Pipeline #
...@@ -33,6 +33,11 @@ Examples: ...@@ -33,6 +33,11 @@ Examples:
import os import os
import sys import sys
import logging
__logging_format__='[%(levelname)s] %(message)s'
logging.basicConfig(format=__logging_format__)
logger = logging.getLogger('bob')
from .eval_threshold import apthres from .eval_threshold import apthres
...@@ -57,6 +62,10 @@ def main(user_input=None): ...@@ -57,6 +62,10 @@ def main(user_input=None):
version=completions['version'], version=completions['version'],
) )
# Sets-up logging
if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
# handles threshold validation # handles threshold validation
try: try:
args['<threshold>'] = float(args['<threshold>']) args['<threshold>'] = float(args['<threshold>'])
......
...@@ -58,7 +58,7 @@ logger = logging.getLogger('bob') ...@@ -58,7 +58,7 @@ logger = logging.getLogger('bob')
def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit): def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
"""Prints a single output line that contains all info for a given criterium""" """Prints a single output line that contains all info for a given criterion"""
from .. import eer_threshold, min_hter_threshold, farfrr from .. import eer_threshold, min_hter_threshold, farfrr
...@@ -73,7 +73,7 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit): ...@@ -73,7 +73,7 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
test_far, test_frr = farfrr(test_neg, test_pos, thres) test_far, test_frr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0 test_hter = (test_far + test_frr)/2.0
print("[Min. criterium: %s] Threshold on Development set: %e" % (crit, thres)) print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres))
dev_ni = dev_neg.shape[0] #number of impostors dev_ni = dev_neg.shape[0] #number of impostors
dev_fa = int(round(dev_far*dev_ni)) #number of false accepts dev_fa = int(round(dev_far*dev_ni)) #number of false accepts
......
...@@ -19,11 +19,11 @@ Options: ...@@ -19,11 +19,11 @@ Options:
-h, --help Shows this help message and exits -h, --help Shows this help message and exits
-V, --version Prints the version and exits -V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level -v, --verbose Increases the output verbosity level
-c <crit>, --criterium=<crit> The minimization criterium to use (choose -c <crit>, --criterion=<crit> The minimization criterion to use (choose
between mhter, mwer or eer) [default: eer] between mhter, mwer or eer) [default: eer]
-w <float>, --cost=<float> The value w of the cost when minimizing using -w <float>, --cost=<float> The value w of the cost when minimizing using
the minimum weighter error rate (mwer) the minimum weighter error rate (mwer)
criterium. This value is ignored for eer or criterion. This value is ignored for eer or
mhter criteria. [default: 0.5] mhter criteria. [default: 0.5]
...@@ -31,11 +31,11 @@ Examples: ...@@ -31,11 +31,11 @@ Examples:
1. Specify a different criteria (only mhter, mwer or eer accepted): 1. Specify a different criteria (only mhter, mwer or eer accepted):
$ %(prog)s --criterium=mhter scores.txt $ %(prog)s --criterion=mhter scores.txt
2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4: 2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4:
$ %(prog)s --criterium=mwer --cost=0.4 scores.txt $ %(prog)s --criterion=mwer --cost=0.4 scores.txt
3. Parse your input using a 5-column format 3. Parse your input using a 5-column format
...@@ -47,6 +47,11 @@ Examples: ...@@ -47,6 +47,11 @@ Examples:
import os import os
import sys import sys
import logging
__logging_format__='[%(levelname)s] %(message)s'
logging.basicConfig(format=__logging_format__)
logger = logging.getLogger('bob')
def apthres(neg, pos, thres): def apthres(neg, pos, thres):
"""Prints a single output line that contains all info for the threshold""" """Prints a single output line that contains all info for the threshold"""
...@@ -102,10 +107,14 @@ def main(user_input=None): ...@@ -102,10 +107,14 @@ def main(user_input=None):
version=completions['version'], version=completions['version'],
) )
# validates criterium # Sets-up logging
if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
# validates criterion
valid_criteria = ('eer', 'mhter', 'mwer') valid_criteria = ('eer', 'mhter', 'mwer')
if args['--criterium'] not in valid_criteria: if args['--criterion'] not in valid_criteria:
raise docopt.DocoptExit("--criterium must be one of %s" % \ raise docopt.DocoptExit("--criterion must be one of %s" % \
', '.join(valid_criteria)) ', '.join(valid_criteria))
# handles cost validation # handles cost validation
...@@ -121,7 +130,7 @@ def main(user_input=None): ...@@ -121,7 +130,7 @@ def main(user_input=None):
from ..load import load_score, get_negatives_positives from ..load import load_score, get_negatives_positives
neg, pos = get_negatives_positives(load_score(args['<scores>'])) neg, pos = get_negatives_positives(load_score(args['<scores>']))
t = calculate(neg, pos, args['--criterium'], args['--cost']) t = calculate(neg, pos, args['--criterion'], args['--cost'])
print("Threshold:", t) print("Threshold:", t)
apthres(neg, pos, t) apthres(neg, pos, t)
......
#!/usr/bin/env python #!/usr/bin/env python
# vim: set fileencoding=utf-8 : # vim: set fileencoding=utf-8 :
# Manuel Guenther <manuel.guenther@idiap.ch>
# Tue Jul 2 14:52:49 CEST 2013 # Tue Jul 2 14:52:49 CEST 2013
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
"""Collects results of verification experiments recursively, reports results
"""
This script parses through the given directory, collects all results of This script parses through the given directory, collects all results of
verification experiments that are stored in file with the given file name. It verification experiments that are stored in file with the given file name. It
supports the split into development and test set of the data, as well as supports the split into development and test set of the data, as well as
...@@ -23,90 +20,62 @@ information are given in columns: ...@@ -23,90 +20,62 @@ information are given in columns:
The measure type of the development set can be changed to compute "HTER" or The measure type of the development set can be changed to compute "HTER" or
"FAR" thresholds instead, using the --criterion option. "FAR" thresholds instead, using the --criterion option.
Usage: %(prog)s [-v...] [options]
%(prog)s --help
%(prog)s --version
Options:
-h, --help Shows this help message and exit
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-d <path>, --devel-name=<path> Name of the file containing the development
scores [default: scores-dev]
-e <path>, --eval-name=<path> Name of the file containing the evaluation
scores [default: scores-eval]
-D <dir>, --directory=<dir> The directory where the results should be
collected from [default: .]
-n <dir>, --nonorm-dir=<dir> Directory where the unnormalized scores are
found [default: nonorm]
-z <dir>, --ztnorm-dir=<dir> Directory where the normalized scores are
found [default: ztnorm]
-s, --sort If set, sorts the results
-k <key>, --sort-key=<key> Sorts the results according to the given key.
May be one of "nonorm_dev", "nonorm_eval",
"ztnorm_dev", "ztnorm_eval" or "dir"
[default: dir]
-c <crit>, --criterion=<crit> Report Equal Rates (EER) rather than Half
Total Error Rate (HTER). Choose between
"HTER", "EER" or "FAR" [default: HTER]
-o <path>, --output=<path> If set, outputs results to a file named after
the option. If not set, writes to the console
""" """
import os import os
import sys import sys
#from apport.hookutils import default
import logging
def get_args(): __logging_format__='[%(levelname)s] %(message)s'
"""Parse the program options""" logging.basicConfig(format=__logging_format__)
logger = logging.getLogger('bob')
import argparse
from .. import load
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-d', '--devel-name', dest="dev", default="scores-dev", metavar="FILE",
help = "Name of the file containing the development scores")
parser.add_argument('-e', '--eval-name', dest="eval", default="scores-eval", metavar="FILE",
help = "Name of the file containing the evaluation scores")
parser.add_argument('-D', '--directory', default=".",
help = "The directory where the results should be collected from.")
parser.add_argument('-n', '--nonorm-dir', dest="nonorm", default="nonorm", metavar = "DIR",
help = "Directory where the unnormalized scores are found")
parser.add_argument('-z', '--ztnorm-dir', dest="ztnorm", default = "ztnorm", metavar = "DIR",
help = "Directory where the normalized scores are found")
parser.add_argument('-s', '--sort', dest="sort", action='store_true',
help = "Sort the results")
parser.add_argument('-k', '--sort-key', dest='key', default = 'dir', choices=['nonorm_dev','nonorm_eval','ztnorm_dev','ztnorm_eval','dir'],
help = "Sort the results accordign to the given key")
parser.add_argument('-c', '--criterion', dest='criterion', default = 'HTER', choices=['HTER', 'EER', 'FAR'],
help = "Report Equal Rates (EER) rather than Half Total Error Rate (HTER)")
parser.add_argument('-o', '--output', dest="output",
help="Name of the output file that will contain the HTER scores")
parser.add_argument('--self-test', action='store_true', help=argparse.SUPPRESS)
parser.add_argument('-p', '--parser', dest="parser", default="4column", metavar="NAME.FUNCTION",
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details")
# parse arguments
args = parser.parse_args()
# parse the score-parser
if args.parser.lower() in ('4column', '4col'):
args.parser = load.split_four_column
elif args.parser.lower() in ('5column', '5col'):
args.parser = load.split_five_column
else: #try an import
if args.parser.find('.') == -1:
parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser)
mod, fct = args.parser.rsplit('.', 2)
import imp
try:
fp, pathname, description = imp.find_module(mod, ['.'] + sys.path)
except Exception as e:
parser.error("import error for '%s': %s" % (args.parser, e))
try:
pmod = imp.load_module(mod, fp, pathname, description)
args.parser = getattr(pmod, fct)
except Exception as e:
parser.error("loading error for '%s': %s" % (args.parser, e))
finally:
fp.close()
return args
class Result: class Result:
def __init__(self, dir, args): def __init__(self, dir, args):
self.dir = dir self.dir = dir
self.m_args = args self.args = args
self.nonorm_dev = None self.nonorm_dev = None
self.nonorm_eval = None self.nonorm_eval = None
self.ztnorm_dev = None self.ztnorm_dev = None
self.ztnorm_eval = None self.ztnorm_eval = None
def __calculate__(self, dev_file, eval_file = None): def __calculate__(self, dev_file, eval_file = None):
from ..load import load_score, get_negatives_positives
from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr
dev_neg, dev_pos = self.m_args.parser(dev_file) dev_neg, dev_pos = get_negatives_positives(load_score(dev_file))
# switch which threshold function to use; # switch which threshold function to use;
# THIS f***ing piece of code really is what python authors propose: # THIS f***ing piece of code really is what python authors propose:
...@@ -114,29 +83,31 @@ class Result: ...@@ -114,29 +83,31 @@ class Result:
'EER' : eer_threshold, 'EER' : eer_threshold,
'HTER' : min_hter_threshold, 'HTER' : min_hter_threshold,
'FAR' : far_threshold 'FAR' : far_threshold
} [self.m_args.criterion](dev_neg, dev_pos) } [self.args['--criterion']](dev_neg, dev_pos)
# compute far and frr for the given threshold # compute far and frr for the given threshold
dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold) dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold)
dev_hter = (dev_far + dev_frr)/2.0 dev_hter = (dev_far + dev_frr)/2.0
if eval_file: if eval_file:
eval_neg, eval_pos = self.m_args.parser(eval_file) eval_neg, eval_pos = get_negatives_positives(load_score(eval_file))
eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold) eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold)
eval_hter = (eval_far + eval_frr)/2.0 eval_hter = (eval_far + eval_frr)/2.0
else: else:
eval_hter = None eval_hter = None
if self.m_args.criterion == 'FAR': if self.args['--criterion'] == 'FAR':
return (dev_frr, eval_frr) return (dev_frr, eval_frr)
else: else:
return (dev_hter, eval_hter) return (dev_hter, eval_hter)
def nonorm(self, dev_file, eval_file = None): def nonorm(self, dev_file, eval_file = None):
(self.nonorm_dev, self.nonorm_eval) = self.__calculate__(dev_file, eval_file) (self.nonorm_dev, self.nonorm_eval) = \
self.__calculate__(dev_file, eval_file)
def ztnorm(self, dev_file, eval_file = None): def ztnorm(self, dev_file, eval_file = None):
(self.ztnorm_dev, self.ztnorm_eval) = self.__calculate__(dev_file, eval_file) (self.ztnorm_dev, self.ztnorm_eval) = \
self.__calculate__(dev_file, eval_file)
def __str__(self): def __str__(self):
str = "" str = ""
...@@ -153,12 +124,14 @@ class Result: ...@@ -153,12 +124,14 @@ class Result:
results = [] results = []
def add_results(args, nonorm, ztnorm = None): def add_results(args, nonorm, ztnorm = None):
r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args) r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args)
print("Adding results from directory", r.dir) print("Adding results from directory", r.dir)
# check if the results files are there # check if the results files are there
dev_file = os.path.join(nonorm, args.dev) dev_file = os.path.join(nonorm, args['--devel-name'])
eval_file = os.path.join(nonorm, args.eval) eval_file = os.path.join(nonorm, args['--eval-name'])
if os.path.isfile(dev_file): if os.path.isfile(dev_file):
if os.path.isfile(eval_file): if os.path.isfile(eval_file):
r.nonorm(dev_file, eval_file) r.nonorm(dev_file, eval_file)
...@@ -166,8 +139,8 @@ def add_results(args, nonorm, ztnorm = None): ...@@ -166,8 +139,8 @@ def add_results(args, nonorm, ztnorm = None):
r.nonorm(dev_file) r.nonorm(dev_file)
if ztnorm: if ztnorm:
dev_file = os.path.join(ztnorm, args.dev) dev_file = os.path.join(ztnorm, args['--devel-name'])
eval_file = os.path.join(ztnorm, args.eval) eval_file = os.path.join(ztnorm, args['--eval-name'])
if os.path.isfile(dev_file): if os.path.isfile(dev_file):
if os.path.isfile(eval_file): if os.path.isfile(eval_file):
r.ztnorm(dev_file, eval_file) r.ztnorm(dev_file, eval_file)
...@@ -176,15 +149,17 @@ def add_results(args, nonorm, ztnorm = None): ...@@ -176,15 +149,17 @@ def add_results(args, nonorm, ztnorm = None):
results.append(r) results.append(r)
def recurse(args, path): def recurse(args, path):
dir_list = os.listdir(path) dir_list = os.listdir(path)
# check if the score directories are included in the current path # check if the score directories are included in the current path
if args.nonorm in dir_list: if args['--nonorm-dir'] in dir_list:
if args.ztnorm in dir_list: if args['--ztnorm-dir'] in dir_list:
add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm)) add_results(args, os.path.join(path, args['--nonorm-dir']),
os.path.join(path, args['--ztnorm-dir']))
else: else:
add_results(args, os.path.join(path, args.nonorm)) add_results(args, os.path.join(path, args['--nonorm-dir']))
for e in dir_list: for e in dir_list:
real_path = os.path.join(path, e) real_path = os.path.join(path, e)
...@@ -199,19 +174,52 @@ def table(): ...@@ -199,19 +174,52 @@ def table():
A += str(r) + "\n" A += str(r) + "\n"
return A return A
def main():
args = get_args()
recurse(args, args.directory) def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO)
elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG)
# checks sort-key
valid_sort_keys = 'nonorm_dev nonorm_eval ztnorm_dev ztnorm_eval dir'.split()
if args['--sort-key'] not in valid_sort_keys:
raise docopt.DocoptExit('--sort-key must be one of %s' % \
', '.join(valid_sort_keys))
# checks criterion
valid_criterion = 'HTER EER FAR'.split()
if args['--criterion'] not in valid_criterion:
raise docopt.DocoptExit('--criterion must be one of %s' % \
', '.join(valid_criterion))
recurse(args, args['--directory'])
if args.sort: if args['--sort']:
import operator import operator
results.sort(key=operator.attrgetter(args.key)) results.sort(key=operator.attrgetter(args['--sort-key']))
if args.self_test: if args['--output']:
_ = table() f = open(args['--output'], "w")
elif args.output:
f = open(args.output, "w")
f.writelines(table()) f.writelines(table())
f.close() f.close()
else: else:
......
...@@ -482,13 +482,13 @@ evaluation and plotting of development and test set data using our combined ...@@ -482,13 +482,13 @@ evaluation and plotting of development and test set data using our combined
.. code-block:: sh .. code-block:: sh
$ bob_compute_perf.py --devel=development-scores-4col.txt --test=test-scores-4col.txt $ bob_compute_perf.py --devel=development-scores-4col.txt --test=test-scores-4col.txt
[Min. criterium: EER] Threshold on Development set: -4.787956e-03 [Min. criterion: EER] Threshold on Development set: -4.787956e-03
| Development | Test | Development | Test
-------+-----------------+------------------ -------+-----------------+------------------
FAR | 6.731% (35/520) | 2.500% (13/520) FAR | 6.731% (35/520) | 2.500% (13/520)
FRR | 6.667% (26/390) | 6.154% (24/390) FRR | 6.667% (26/390) | 6.154% (24/390)
HTER | 6.699% | 4.327% HTER | 6.699% | 4.327%
[Min. criterium: Min. HTER] Threshold on Development set: 3.411070e-03 [Min. criterion: Min. HTER] Threshold on Development set: 3.411070e-03
| Development | Test | Development | Test
-------+-----------------+------------------ -------+-----------------+------------------
FAR | 4.231% (22/520) | 1.923% (10/520) FAR | 4.231% (22/520) | 1.923% (10/520)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment