Skip to content

bob.measure
bob.measure
Commit 31f300d4 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files
Options

Remove bio related files

parent c89e5cb3
Pipeline #17697 canceled with stage
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing with 0 additions and 2345 deletions
    bob/measure/load.py deleted 100644 → 0
    View file @ c89e5cb3
    This diff is collapsed.
    bob/measure/openbr.py deleted 100644 → 0
    View file @ c89e5cb3
    This diff is collapsed.
    bob/measure/script/apply_threshold.py deleted 100644 → 0
    View file @ c89e5cb3
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Wed 28 Sep 2016 17:55:17 CEST
    """Applies a threshold to score file and reports error rates
    Usage: %(prog)s [-v...] [options] [--] <threshold> <scores>
    %(prog)s --help
    %(prog)s --version
    Arguments:
    <threshold> The threshold value to apply (float)
    <scores> Path to the file containing the scores where to apply the
    threshold and calculate error rates
    Options:
    -h, --help Shows this help message and exits
    -V, --version Prints the version and exits
    -v, --verbose Increases the output verbosity level
    Examples:
    Applies the threshold of 0.5 to the scores file in scores.txt and reports:
    $ %(prog)s 0.5 scores.txt
    """
    import os
    import sys
    from .eval_threshold import apthres
    import bob.core
    logger = bob.core.log.setup("bob.measure")
    def main(user_input=None):
    if user_input is not None:
    argv = user_input
    else:
    argv = sys.argv[1:]
    import docopt
    import pkg_resources
    completions = dict(
    prog=os.path.basename(sys.argv[0]),
    version=pkg_resources.require('bob.measure')[0].version)
    args = docopt.docopt(
    __doc__ % completions,
    argv=argv,
    version=completions['version'],)
    # Sets-up logging
    verbosity = int(args['--verbose'])
    bob.core.log.set_verbosity_level(logger, verbosity)
    # handles threshold validation
    try:
    args['<threshold>'] = float(args['<threshold>'])
    except Exception:
    raise docopt.DocoptExit("cannot convert %s into float for threshold" %
    args['<threshold>'])
    from ..load import load_score, get_negatives_positives
    neg, pos = get_negatives_positives(load_score(args['<scores>']))
    apthres(neg, pos, args['<threshold>'])
    return 0
    bob/measure/script/compute_perf.py deleted 100644 → 0
    View file @ c89e5cb3
    This diff is collapsed.
    bob/measure/script/eval_threshold.py deleted 100644 → 0
    View file @ c89e5cb3
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Wed 28 Sep 2016 16:56:52 CEST
    """Computes the threshold following a minimization criteria on input scores
    Usage: %(prog)s [-v...] [options] <scores>
    %(prog)s --help
    %(prog)s --version
    Arguments:
    <scores> Path to the file containing the scores to be used for calculating
    the threshold
    Options:
    -h, --help Shows this help message and exits
    -V, --version Prints the version and exits
    -v, --verbose Increases the output verbosity level
    -c <crit>, --criterion=<crit> The minimization criterion to use (choose
    between mhter, mwer or eer) [default: eer]
    -w <float>, --cost=<float> The value w of the cost when minimizing using
    the minimum weighter error rate (mwer)
    criterion. This value is ignored for eer or
    mhter criteria. [default: 0.5]
    Examples:
    1. Specify a different criteria (only mhter, mwer or eer accepted):
    $ %(prog)s --criterion=mhter scores.txt
    2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4:
    $ %(prog)s --criterion=mwer --cost=0.4 scores.txt
    3. Parse your input using a 5-column format
    $ %(prog)s scores.txt
    """
    import os
    import sys
    import bob.core
    logger = bob.core.log.setup("bob.measure")
    def apthres(neg, pos, thres):
    """Prints a single output line that contains all info for the threshold"""
    from .. import farfrr
    far, frr = farfrr(neg, pos, thres)
    hter = (far + frr)/2.0
    ni = neg.shape[0] #number of impostors
    fa = int(round(far*ni)) #number of false accepts
    nc = pos.shape[0] #number of clients
    fr = int(round(frr*nc)) #number of false rejects
    print("FAR : %.3f%% (%d/%d)" % (100*far, fa, ni))
    print("FRR : %.3f%% (%d/%d)" % (100*frr, fr, nc))
    print("HTER: %.3f%%" % (100*hter,))
    def calculate(neg, pos, crit, cost):
    """Returns the threshold given a certain criteria"""
    if crit == 'eer':
    from .. import eer_threshold
    return eer_threshold(neg, pos)
    elif crit == 'mhter':
    from .. import min_hter_threshold
    return min_hter_threshold(neg, pos)
    # defaults to the minimum of the weighter error rate
    from .. import min_weighted_error_rate_threshold
    return min_weighted_error_rate_threshold(neg, pos, cost)
    def main(user_input=None):
    if user_input is not None:
    argv = user_input
    else:
    argv = sys.argv[1:]
    import docopt
    import pkg_resources
    completions = dict(
    prog=os.path.basename(sys.argv[0]),
    version=pkg_resources.require('bob.measure')[0].version
    )
    args = docopt.docopt(
    __doc__ % completions,
    argv=argv,
    version=completions['version'],
    )
    # Sets-up logging
    verbosity = int(args['--verbose'])
    bob.core.log.set_verbosity_level(logger, verbosity)
    # validates criterion
    valid_criteria = ('eer', 'mhter', 'mwer')
    if args['--criterion'] not in valid_criteria:
    raise docopt.DocoptExit("--criterion must be one of %s" % \
    ', '.join(valid_criteria))
    # handles cost validation
    try:
    args['--cost'] = float(args['--cost'])
    except:
    raise docopt.DocoptExit("cannot convert %s into float for cost" % \
    args['--cost'])
    if args['--cost'] < 0.0 or args['--cost'] > 1.0:
    docopt.DocoptExit("cost should lie between 0.0 and 1.0")
    from ..load import load_score, get_negatives_positives
    neg, pos = get_negatives_positives(load_score(args['<scores>']))
    t = calculate(neg, pos, args['--criterion'], args['--cost'])
    print("Threshold:", t)
    apthres(neg, pos, t)
    return 0
    bob/measure/script/plot_cmc.py deleted 100644 → 0
    View file @ c89e5cb3
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Wed 28 Sep 2016 21:24:46 CEST
    """Computes and plots a cumulative rank characteristics (CMC) curve
    Usage: %(prog)s [-v...] [options] <scores>
    %(prog)s --help
    %(prog)s --version
    Arguments:
    <scores> The score file in 4 or 5 column format to test
    Options:
    -h, --help Shows this help message and exits
    -V, --version Prints the version and exits
    -v, --verbose Increases the output verbosity level
    -o <path>, --output=<path> Name of the output file that will contain the
    plots [default: cmc.pdf]
    -x, --no-plot If set, then I'll execute no plotting
    -l, --log-x-scale If set, plots logarithmic rank axis
    -r <int>, --rank=<int> Plot detection & identification rate curve for
    the given rank instead of the CMC curve.
    """
    from __future__ import print_function
    import os
    import sys
    import bob.core
    logger = bob.core.log.setup("bob.measure")
    def main(user_input=None):
    if user_input is not None:
    argv = user_input
    else:
    argv = sys.argv[1:]
    import docopt
    import pkg_resources
    completions = dict(
    prog=os.path.basename(sys.argv[0]),
    version=pkg_resources.require('bob.measure')[0].version
    )
    args = docopt.docopt(
    __doc__ % completions,
    argv=argv,
    version=completions['version'],
    )
    # Sets-up logging
    verbosity = int(args['--verbose'])
    bob.core.log.set_verbosity_level(logger, verbosity)
    # Validates rank
    if args['--rank'] is not None:
    try:
    args['--rank'] = int(args['--rank'])
    except:
    raise docopt.DocoptExit("cannot convert %s into int for rank" % \
    args['--rank'])
    if args['--rank'] <= 0:
    raise docopt.DocoptExit('Rank (--rank) should greater than zero')
    from .. import load
    # Loads score file
    data = load.cmc(args['<scores>'])
    # compute recognition rate
    from .. import recognition_rate
    if args['--rank'] is not None:
    rr = recognition_rate(data, rank=args['--rank'])
    else:
    rr = recognition_rate(data)
    print("Recognition rate for score file %s is %3.2f%%" % (args['<scores>'],
    rr * 100))
    if not args['--no-plot']:
    from .. import plot
    # compute CMC
    import matplotlib
    if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
    import matplotlib.pyplot as mpl
    from matplotlib.backends.backend_pdf import PdfPages
    pp = PdfPages(args['--output'])
    # CMC
    fig = mpl.figure()
    if args['--rank'] is None:
    max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2),
    logx = args['--log-x-scale'])
    mpl.title("CMC Curve")
    if args['--log-x-scale']:
    mpl.xlabel('Rank (log)')
    else:
    mpl.xlabel('Rank')
    mpl.ylabel('Recognition Rate in %')
    ticks = [int(t) for t in mpl.xticks()[0]]
    mpl.xticks(ticks, ticks)
    mpl.xlim([1, max_rank])
    else:
    plot.detection_identification_curve(data, rank = args['--rank'],
    color=(0,0,1), linestyle='--', dashes=(6,2),
    logx = args['--log-x-scale'])
    mpl.title("Detection \& Identification Curve")
    if args['--log-x-scale']:
    mpl.xlabel('False Acceptance Rate (log) in %')
    else:
    mpl.xlabel('False Acceptance Rate in %')
    mpl.ylabel('Detection \& Identification Rate in %')
    ticks = ["%s"%(t*100) for t in mpl.xticks()[0]]
    mpl.xticks(mpl.xticks()[0], ticks)
    mpl.xlim([1e-4, 1])
    mpl.grid(True, color=(0.3,0.3,0.3))
    mpl.ylim(ymax=101)
    # convert log-scale ticks to normal numbers
    pp.savefig(fig)
    pp.close()
    return 0
    bob/measure/script/recurse_results.py deleted 100644 → 0
    View file @ c89e5cb3
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Tue Jul 2 14:52:49 CEST 2013
    """Collects results of verification experiments recursively, reports results
    This script parses through the given directory, collects all results of
    verification experiments that are stored in file with the given file name. It
    supports the split into development and test set of the data, as well as
    ZT-normalized scores.
    All result files are parsed and evaluated. For each directory, the following
    information are given in columns:
    * The Equal Error Rate of the development set
    * The Equal Error Rate of the development set after ZT-Normalization
    * The Half Total Error Rate of the evaluation set
    * The Half Total Error Rate of the evaluation set after ZT-Normalization
    * The sub-directory where the scores can be found
    The measure type of the development set can be changed to compute "HTER" or
    "FAR" thresholds instead, using the --criterion option.
    Usage: %(prog)s [-v...] [options]
    %(prog)s --help
    %(prog)s --version
    Options:
    -h, --help Shows this help message and exit
    -V, --version Prints the version and exits
    -v, --verbose Increases the output verbosity level
    -d <path>, --devel-name=<path> Name of the file containing the development
    scores [default: scores-dev]
    -e <path>, --eval-name=<path> Name of the file containing the evaluation
    scores [default: scores-eval]
    -D <dir>, --directory=<dir> The directory where the results should be
    collected from [default: .]
    -n <dir>, --nonorm-dir=<dir> Directory where the unnormalized scores are
    found [default: nonorm]
    -z <dir>, --ztnorm-dir=<dir> Directory where the normalized scores are
    found [default: ztnorm]
    -s, --sort If set, sorts the results
    -k <key>, --sort-key=<key> Sorts the results according to the given key.
    May be one of "nonorm_dev", "nonorm_eval",
    "ztnorm_dev", "ztnorm_eval" or "dir"
    [default: dir]
    -c <crit>, --criterion=<crit> Report Equal Rates (EER) rather than Half
    Total Error Rate (HTER). Choose between
    "HTER", "EER" or "FAR" [default: HTER]
    -o <path>, --output=<path> If set, outputs results to a file named after
    the option. If not set, writes to the console
    """
    import os
    import sys
    import bob.core
    logger = bob.core.log.setup("bob.measure")
    class Result:
    def __init__(self, dir, args):
    self.dir = dir
    self.args = args
    self.nonorm_dev = None
    self.nonorm_eval = None
    self.ztnorm_dev = None
    self.ztnorm_eval = None
    def __calculate__(self, dev_file, eval_file = None):
    from ..load import load_score, get_negatives_positives
    from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr
    dev_neg, dev_pos = get_negatives_positives(load_score(dev_file))
    # switch which threshold function to use;
    # THIS f***ing piece of code really is what python authors propose:
    threshold = {
    'EER' : eer_threshold,
    'HTER' : min_hter_threshold,
    'FAR' : far_threshold
    } [self.args['--criterion']](dev_neg, dev_pos)
    # compute far and frr for the given threshold
    dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold)
    dev_hter = (dev_far + dev_frr)/2.0
    if eval_file:
    eval_neg, eval_pos = get_negatives_positives(load_score(eval_file))
    eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold)
    eval_hter = (eval_far + eval_frr)/2.0
    else:
    eval_hter = None
    if self.args['--criterion'] == 'FAR':
    return (dev_frr, eval_frr)
    else:
    return (dev_hter, eval_hter)
    def nonorm(self, dev_file, eval_file = None):
    (self.nonorm_dev, self.nonorm_eval) = \
    self.__calculate__(dev_file, eval_file)
    def ztnorm(self, dev_file, eval_file = None):
    (self.ztnorm_dev, self.ztnorm_eval) = \
    self.__calculate__(dev_file, eval_file)
    def __str__(self):
    str = ""
    for v in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]:
    if v:
    val = "% 2.3f%%"%(v*100)
    else:
    val = "None"
    cnt = 16-len(val)
    str += " "*cnt + val
    str += " %s"%self.dir
    return str[5:]
    results = []
    def add_results(args, nonorm, ztnorm = None):
    r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args)
    print("Adding results from directory", r.dir)
    # check if the results files are there
    dev_file = os.path.join(nonorm, args['--devel-name'])
    eval_file = os.path.join(nonorm, args['--eval-name'])
    if os.path.isfile(dev_file):
    if os.path.isfile(eval_file):
    r.nonorm(dev_file, eval_file)
    else:
    r.nonorm(dev_file)
    if ztnorm:
    dev_file = os.path.join(ztnorm, args['--devel-name'])
    eval_file = os.path.join(ztnorm, args['--eval-name'])
    if os.path.isfile(dev_file):
    if os.path.isfile(eval_file):
    r.ztnorm(dev_file, eval_file)
    else:
    r.ztnorm(dev_file)
    results.append(r)
    def recurse(args, path):
    dir_list = os.listdir(path)
    # check if the score directories are included in the current path
    if args['--nonorm-dir'] in dir_list:
    if args['--ztnorm-dir'] in dir_list:
    add_results(args, os.path.join(path, args['--nonorm-dir']),
    os.path.join(path, args['--ztnorm-dir']))
    else:
    add_results(args, os.path.join(path, args['--nonorm-dir']))
    for e in dir_list:
    real_path = os.path.join(path, e)
    if os.path.isdir(real_path):
    recurse(args, real_path)
    def table():
    A = " "*2 + 'dev nonorm'+ " "*5 + 'dev ztnorm' + " "*6 + 'eval nonorm' + " "*4 + 'eval ztnorm' + " "*12 + 'directory\n'
    A += "-"*100+"\n"
    for r in results:
    A += str(r) + "\n"
    return A
    def main(user_input=None):
    if user_input is not None:
    argv = user_input
    else:
    argv = sys.argv[1:]
    import docopt
    import pkg_resources
    completions = dict(
    prog=os.path.basename(sys.argv[0]),
    version=pkg_resources.require('bob.measure')[0].version
    )
    args = docopt.docopt(
    __doc__ % completions,
    argv=argv,
    version=completions['version'],
    )
    # Sets-up logging
    verbosity = int(args['--verbose'])
    bob.core.log.set_verbosity_level(logger, verbosity)
    # checks sort-key
    valid_sort_keys = 'nonorm_dev nonorm_eval ztnorm_dev ztnorm_eval dir'.split()
    if args['--sort-key'] not in valid_sort_keys:
    raise docopt.DocoptExit('--sort-key must be one of %s' % \
    ', '.join(valid_sort_keys))
    # checks criterion
    valid_criterion = 'HTER EER FAR'.split()
    if args['--criterion'] not in valid_criterion:
    raise docopt.DocoptExit('--criterion must be one of %s' % \
    ', '.join(valid_criterion))
    recurse(args, args['--directory'])
    if args['--sort']:
    import operator
    results.sort(key=operator.attrgetter(args['--sort-key']))
    if args['--output']:
    f = open(args['--output'], "w")
    f.writelines(table())
    f.close()
    else:
    print(table())
    bob/measure/test_io.py deleted 100644 → 0
    View file @ c89e5cb3
    This diff is collapsed.
    bob/measure/test_scripts.py deleted 100644 → 0
    View file @ c89e5cb3
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Tue 21 Aug 2012 12:14:43 CEST
    """Script tests for bob.measure
    """
    import os
    import tempfile
    import nose.tools
    import pkg_resources
    def F(f):
    """Returns the test file on the "data" subdirectory"""
    return pkg_resources.resource_filename(__name__, os.path.join('data', f))
    DEV_SCORES = F('dev-4col.txt')
    TEST_SCORES = F('test-4col.txt')
    DEV_SCORES_5COL = F('dev-5col.txt')
    TEST_SCORES_5COL = F('test-5col.txt')
    SCORES_4COL_CMC = F('scores-cmc-4col.txt')
    SCORES_5COL_CMC = F('scores-cmc-5col.txt')
    SCORES_4COL_CMC_OS = F('scores-cmc-4col-open-set.txt')
    def test_compute_perf():
    # sanity checks
    assert os.path.exists(DEV_SCORES)
    assert os.path.exists(TEST_SCORES)
    tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
    cmdline = [
    DEV_SCORES,
    TEST_SCORES,
    '--output=' + tmp_output.name,
    ]
    from .script.compute_perf import main
    nose.tools.eq_(main(cmdline), 0)
    tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.txt')
    cmdline = [
    DEV_SCORES,
    TEST_SCORES,
    '-x',
    '--log=' + tmp_output.name,
    ]
    nose.tools.eq_(main(cmdline), 0)
    nose.tools.eq_(
    open(tmp_output.name).read(),
    open(F('compute_perf.output.txt')).read())
    def test_compute_perf_only_dev():
    # sanity checks
    assert os.path.exists(DEV_SCORES)
    tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
    cmdline = [
    DEV_SCORES,
    '--output=' + tmp_output.name,
    ]
    from .script.compute_perf import main
    nose.tools.eq_(main(cmdline), 0)
    def test_eval_threshold():
    # sanity checks
    assert os.path.exists(DEV_SCORES)
    cmdline = [DEV_SCORES]
    from .script.eval_threshold import main
    nose.tools.eq_(main(cmdline), 0)
    def test_apply_threshold():
    # sanity checks
    assert os.path.exists(TEST_SCORES)
    cmdline = [
    '0.5',
    TEST_SCORES,
    ]
    from .script.apply_threshold import main
    nose.tools.eq_(main(cmdline), 0)
    cmdline = [
    '--',
    '-0.5',
    TEST_SCORES,
    ]
    nose.tools.eq_(main(cmdline), 0)
    def test_compute_perf_5col():
    # sanity checks
    assert os.path.exists(DEV_SCORES_5COL)
    assert os.path.exists(TEST_SCORES_5COL)
    tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
    cmdline = [
    DEV_SCORES_5COL,
    TEST_SCORES_5COL,
    '--output=' + tmp_output.name,
    ]
    from .script.compute_perf import main
    nose.tools.eq_(main(cmdline), 0)