Commit 31f300d4 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Remove bio related files

parent c89e5cb3
Pipeline #17697 canceled with stage
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Wed 28 Sep 2016 17:55:17 CEST
"""Applies a threshold to score file and reports error rates
Usage: %(prog)s [-v...] [options] [--] <threshold> <scores>
%(prog)s --help
%(prog)s --version
Arguments:
<threshold> The threshold value to apply (float)
<scores> Path to the file containing the scores where to apply the
threshold and calculate error rates
Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
Examples:
Applies the threshold of 0.5 to the scores file in scores.txt and reports:
$ %(prog)s 0.5 scores.txt
"""
import os
import sys
from .eval_threshold import apthres
import bob.core
logger = bob.core.log.setup("bob.measure")
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# handles threshold validation
try:
args['<threshold>'] = float(args['<threshold>'])
except Exception:
raise docopt.DocoptExit("cannot convert %s into float for threshold" %
args['<threshold>'])
from ..load import load_score, get_negatives_positives
neg, pos = get_negatives_positives(load_score(args['<scores>']))
apthres(neg, pos, args['<threshold>'])
return 0
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Wed 28 Sep 2016 16:56:52 CEST
"""Computes the threshold following a minimization criteria on input scores
Usage: %(prog)s [-v...] [options] <scores>
%(prog)s --help
%(prog)s --version
Arguments:
<scores> Path to the file containing the scores to be used for calculating
the threshold
Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-c <crit>, --criterion=<crit> The minimization criterion to use (choose
between mhter, mwer or eer) [default: eer]
-w <float>, --cost=<float> The value w of the cost when minimizing using
the minimum weighter error rate (mwer)
criterion. This value is ignored for eer or
mhter criteria. [default: 0.5]
Examples:
1. Specify a different criteria (only mhter, mwer or eer accepted):
$ %(prog)s --criterion=mhter scores.txt
2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4:
$ %(prog)s --criterion=mwer --cost=0.4 scores.txt
3. Parse your input using a 5-column format
$ %(prog)s scores.txt
"""
import os
import sys
import bob.core
logger = bob.core.log.setup("bob.measure")
def apthres(neg, pos, thres):
"""Prints a single output line that contains all info for the threshold"""
from .. import farfrr
far, frr = farfrr(neg, pos, thres)
hter = (far + frr)/2.0
ni = neg.shape[0] #number of impostors
fa = int(round(far*ni)) #number of false accepts
nc = pos.shape[0] #number of clients
fr = int(round(frr*nc)) #number of false rejects
print("FAR : %.3f%% (%d/%d)" % (100*far, fa, ni))
print("FRR : %.3f%% (%d/%d)" % (100*frr, fr, nc))
print("HTER: %.3f%%" % (100*hter,))
def calculate(neg, pos, crit, cost):
"""Returns the threshold given a certain criteria"""
if crit == 'eer':
from .. import eer_threshold
return eer_threshold(neg, pos)
elif crit == 'mhter':
from .. import min_hter_threshold
return min_hter_threshold(neg, pos)
# defaults to the minimum of the weighter error rate
from .. import min_weighted_error_rate_threshold
return min_weighted_error_rate_threshold(neg, pos, cost)
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# validates criterion
valid_criteria = ('eer', 'mhter', 'mwer')
if args['--criterion'] not in valid_criteria:
raise docopt.DocoptExit("--criterion must be one of %s" % \
', '.join(valid_criteria))
# handles cost validation
try:
args['--cost'] = float(args['--cost'])
except:
raise docopt.DocoptExit("cannot convert %s into float for cost" % \
args['--cost'])
if args['--cost'] < 0.0 or args['--cost'] > 1.0:
docopt.DocoptExit("cost should lie between 0.0 and 1.0")
from ..load import load_score, get_negatives_positives
neg, pos = get_negatives_positives(load_score(args['<scores>']))
t = calculate(neg, pos, args['--criterion'], args['--cost'])
print("Threshold:", t)
apthres(neg, pos, t)
return 0
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Wed 28 Sep 2016 21:24:46 CEST
"""Computes and plots a cumulative rank characteristics (CMC) curve
Usage: %(prog)s [-v...] [options] <scores>
%(prog)s --help
%(prog)s --version
Arguments:
<scores> The score file in 4 or 5 column format to test
Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-o <path>, --output=<path> Name of the output file that will contain the
plots [default: cmc.pdf]
-x, --no-plot If set, then I'll execute no plotting
-l, --log-x-scale If set, plots logarithmic rank axis
-r <int>, --rank=<int> Plot detection & identification rate curve for
the given rank instead of the CMC curve.
"""
from __future__ import print_function
import os
import sys
import bob.core
logger = bob.core.log.setup("bob.measure")
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# Validates rank
if args['--rank'] is not None:
try:
args['--rank'] = int(args['--rank'])
except:
raise docopt.DocoptExit("cannot convert %s into int for rank" % \
args['--rank'])
if args['--rank'] <= 0:
raise docopt.DocoptExit('Rank (--rank) should greater than zero')
from .. import load
# Loads score file
data = load.cmc(args['<scores>'])
# compute recognition rate
from .. import recognition_rate
if args['--rank'] is not None:
rr = recognition_rate(data, rank=args['--rank'])
else:
rr = recognition_rate(data)
print("Recognition rate for score file %s is %3.2f%%" % (args['<scores>'],
rr * 100))
if not args['--no-plot']:
from .. import plot
# compute CMC
import matplotlib
if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages(args['--output'])
# CMC
fig = mpl.figure()
if args['--rank'] is None:
max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2),
logx = args['--log-x-scale'])
mpl.title("CMC Curve")
if args['--log-x-scale']:
mpl.xlabel('Rank (log)')
else:
mpl.xlabel('Rank')
mpl.ylabel('Recognition Rate in %')
ticks = [int(t) for t in mpl.xticks()[0]]
mpl.xticks(ticks, ticks)
mpl.xlim([1, max_rank])
else:
plot.detection_identification_curve(data, rank = args['--rank'],
color=(0,0,1), linestyle='--', dashes=(6,2),
logx = args['--log-x-scale'])
mpl.title("Detection \& Identification Curve")
if args['--log-x-scale']:
mpl.xlabel('False Acceptance Rate (log) in %')
else:
mpl.xlabel('False Acceptance Rate in %')
mpl.ylabel('Detection \& Identification Rate in %')
ticks = ["%s"%(t*100) for t in mpl.xticks()[0]]
mpl.xticks(mpl.xticks()[0], ticks)
mpl.xlim([1e-4, 1])
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.ylim(ymax=101)
# convert log-scale ticks to normal numbers
pp.savefig(fig)
pp.close()
return 0
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tue Jul 2 14:52:49 CEST 2013
"""Collects results of verification experiments recursively, reports results
This script parses through the given directory, collects all results of
verification experiments that are stored in file with the given file name. It
supports the split into development and test set of the data, as well as
ZT-normalized scores.
All result files are parsed and evaluated. For each directory, the following
information are given in columns:
* The Equal Error Rate of the development set
* The Equal Error Rate of the development set after ZT-Normalization
* The Half Total Error Rate of the evaluation set
* The Half Total Error Rate of the evaluation set after ZT-Normalization
* The sub-directory where the scores can be found
The measure type of the development set can be changed to compute "HTER" or
"FAR" thresholds instead, using the --criterion option.
Usage: %(prog)s [-v...] [options]
%(prog)s --help
%(prog)s --version
Options:
-h, --help Shows this help message and exit
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-d <path>, --devel-name=<path> Name of the file containing the development
scores [default: scores-dev]
-e <path>, --eval-name=<path> Name of the file containing the evaluation
scores [default: scores-eval]
-D <dir>, --directory=<dir> The directory where the results should be
collected from [default: .]
-n <dir>, --nonorm-dir=<dir> Directory where the unnormalized scores are
found [default: nonorm]
-z <dir>, --ztnorm-dir=<dir> Directory where the normalized scores are
found [default: ztnorm]
-s, --sort If set, sorts the results
-k <key>, --sort-key=<key> Sorts the results according to the given key.
May be one of "nonorm_dev", "nonorm_eval",
"ztnorm_dev", "ztnorm_eval" or "dir"
[default: dir]
-c <crit>, --criterion=<crit> Report Equal Rates (EER) rather than Half
Total Error Rate (HTER). Choose between
"HTER", "EER" or "FAR" [default: HTER]
-o <path>, --output=<path> If set, outputs results to a file named after
the option. If not set, writes to the console
"""
import os
import sys
import bob.core
logger = bob.core.log.setup("bob.measure")
class Result:
def __init__(self, dir, args):
self.dir = dir
self.args = args
self.nonorm_dev = None
self.nonorm_eval = None
self.ztnorm_dev = None
self.ztnorm_eval = None
def __calculate__(self, dev_file, eval_file = None):
from ..load import load_score, get_negatives_positives
from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr
dev_neg, dev_pos = get_negatives_positives(load_score(dev_file))
# switch which threshold function to use;
# THIS f***ing piece of code really is what python authors propose:
threshold = {
'EER' : eer_threshold,
'HTER' : min_hter_threshold,
'FAR' : far_threshold
} [self.args['--criterion']](dev_neg, dev_pos)
# compute far and frr for the given threshold
dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold)
dev_hter = (dev_far + dev_frr)/2.0
if eval_file:
eval_neg, eval_pos = get_negatives_positives(load_score(eval_file))
eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold)
eval_hter = (eval_far + eval_frr)/2.0
else:
eval_hter = None
if self.args['--criterion'] == 'FAR':
return (dev_frr, eval_frr)
else:
return (dev_hter, eval_hter)
def nonorm(self, dev_file, eval_file = None):
(self.nonorm_dev, self.nonorm_eval) = \
self.__calculate__(dev_file, eval_file)
def ztnorm(self, dev_file, eval_file = None):
(self.ztnorm_dev, self.ztnorm_eval) = \
self.__calculate__(dev_file, eval_file)
def __str__(self):
str = ""
for v in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]:
if v:
val = "% 2.3f%%"%(v*100)
else:
val = "None"
cnt = 16-len(val)
str += " "*cnt + val
str += " %s"%self.dir
return str[5:]
results = []
def add_results(args, nonorm, ztnorm = None):
r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args)
print("Adding results from directory", r.dir)
# check if the results files are there
dev_file = os.path.join(nonorm, args['--devel-name'])
eval_file = os.path.join(nonorm, args['--eval-name'])
if os.path.isfile(dev_file):
if os.path.isfile(eval_file):
r.nonorm(dev_file, eval_file)
else:
r.nonorm(dev_file)
if ztnorm:
dev_file = os.path.join(ztnorm, args['--devel-name'])
eval_file = os.path.join(ztnorm, args['--eval-name'])
if os.path.isfile(dev_file):
if os.path.isfile(eval_file):
r.ztnorm(dev_file, eval_file)
else:
r.ztnorm(dev_file)
results.append(r)
def recurse(args, path):
dir_list = os.listdir(path)
# check if the score directories are included in the current path
if args['--nonorm-dir'] in dir_list:
if args['--ztnorm-dir'] in dir_list:
add_results(args, os.path.join(path, args['--nonorm-dir']),
os.path.join(path, args['--ztnorm-dir']))
else:
add_results(args, os.path.join(path, args['--nonorm-dir']))
for e in dir_list:
real_path = os.path.join(path, e)
if os.path.isdir(real_path):
recurse(args, real_path)
def table():
A = " "*2 + 'dev nonorm'+ " "*5 + 'dev ztnorm' + " "*6 + 'eval nonorm' + " "*4 + 'eval ztnorm' + " "*12 + 'directory\n'
A += "-"*100+"\n"
for r in results:
A += str(r) + "\n"
return A
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.measure')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# checks sort-key
valid_sort_keys = 'nonorm_dev nonorm_eval ztnorm_dev ztnorm_eval dir'.split()
if args['--sort-key'] not in valid_sort_keys:
raise docopt.DocoptExit('--sort-key must be one of %s' % \
', '.join(valid_sort_keys))
# checks criterion
valid_criterion = 'HTER EER FAR'.split()
if args['--criterion'] not in valid_criterion:
raise docopt.DocoptExit('--criterion must be one of %s' % \
', '.join(valid_criterion))
recurse(args, args['--directory'])
if args['--sort']:
import operator
results.sort(key=operator.attrgetter(args['--sort-key']))
if args['--output']:
f = open(args['--output'], "w")
f.writelines(table())
f.close()
else:
print(table())
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tue 21 Aug 2012 12:14:43 CEST
"""Script tests for bob.measure
"""
import os
import tempfile
import nose.tools
import pkg_resources
def F(f):
"""Returns the test file on the "data" subdirectory"""
return pkg_resources.resource_filename(__name__, os.path.join('data', f))
DEV_SCORES = F('dev-4col.txt')
TEST_SCORES = F('test-4col.txt')
DEV_SCORES_5COL = F('dev-5col.txt')
TEST_SCORES_5COL = F('test-5col.txt')
SCORES_4COL_CMC = F('scores-cmc-4col.txt')
SCORES_5COL_CMC = F('scores-cmc-5col.txt')
SCORES_4COL_CMC_OS = F('scores-cmc-4col-open-set.txt')
def test_compute_perf():
# sanity checks
assert os.path.exists(DEV_SCORES)
assert os.path.exists(TEST_SCORES)
tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
cmdline = [
DEV_SCORES,
TEST_SCORES,
'--output=' + tmp_output.name,
]
from .script.compute_perf import main
nose.tools.eq_(main(cmdline), 0)
tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.txt')
cmdline = [
DEV_SCORES,
TEST_SCORES,
'-x',
'--log=' + tmp_output.name,
]
nose.tools.eq_(main(cmdline), 0)
nose.tools.eq_(
open(tmp_output.name).read(),
open(F('compute_perf.output.txt')).read())
def test_compute_perf_only_dev():
# sanity checks
assert os.path.exists(DEV_SCORES)
tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
cmdline = [
DEV_SCORES,
'--output=' + tmp_output.name,
]
from .script.compute_perf import main
nose.tools.eq_(main(cmdline), 0)
def test_eval_threshold():
# sanity checks
assert os.path.exists(DEV_SCORES)
cmdline = [DEV_SCORES]
from .script.eval_threshold import main
nose.tools.eq_(main(cmdline), 0)
def test_apply_threshold():
# sanity checks
assert os.path.exists(TEST_SCORES)
cmdline = [
'0.5',
TEST_SCORES,
]
from .script.apply_threshold import main
nose.tools.eq_(main(cmdline), 0)
cmdline = [
'--',
'-0.5',
TEST_SCORES,
]
nose.tools.eq_(main(cmdline), 0)
def test_compute_perf_5col():
# sanity checks
assert os.path.exists(DEV_SCORES_5COL)
assert os.path.exists(TEST_SCORES_5COL)
tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf')
cmdline = [
DEV_SCORES_5COL,
TEST_SCORES_5COL,
'--output=' + tmp_output.name,
]
from .script.compute_perf import main
nose.tools.eq_(main(cmdline), 0)