Commit 5c3475cd authored by André Anjos's avatar André Anjos 💬

Updates; Preparation for C++ bindings

parent 51ade767
[submodule "src/xbob.blitz"]
path = src/xbob.blitz
url = git@github.com:anjos/xbob.blitz
[submodule "src/xbob.math"]
path = src/xbob.math
url = git@github.com:anjos/xbob.math
[submodule "src/xbob.io"]
path = src/xbob.io
url = git@github.com:anjos/xbob.io
......@@ -3,7 +3,7 @@
; Mon 16 Apr 08:29:18 2012 CEST
[buildout]
parts = xbob.blitz xbob.measure scripts
parts = xbob.blitz xbob.math xbob.io xbob.measure scripts
eggs = xbob.measure
ipdb
extensions = mr.developer
......@@ -21,6 +21,16 @@ recipe = xbob.buildout:develop
setup = src/xbob.blitz
eggs = xbob.buildout xbob.extension
[xbob.math]
recipe = xbob.buildout:develop
setup = src/xbob.math
eggs = xbob.blitz xbob.buildout xbob.extension
[xbob.io]
recipe = xbob.buildout:develop
setup = src/xbob.io
eggs = xbob.blitz xbob.buildout xbob.extension
[xbob.measure]
recipe = xbob.buildout:develop
eggs = xbob.blitz
......
......@@ -28,7 +28,8 @@ setup(
install_requires=[
'setuptools',
'xbob.blitz',
#'xbob.math',
'xbob.math',
'xbob.io',
],
namespace_packages=[
......
Subproject commit 6df5905976b1ea66ed2cc1550fd944a2dc939f19
Subproject commit 224853f72bee58d9dc075ee8fc3f228c89124392
Subproject commit 15268b5656de215f23c90ffdeb791c22cbc46509
Subproject commit c5ea8b026b195c0da1da4b090d5061dcbceb59f2
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Wed 11 Dec 15:11:44 2013 CET
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
from ._library import __version__
from . import plot
......@@ -66,15 +73,19 @@ def relevance (input, machine):
return retval
def recognition_rate(cmc_scores):
"""Calculates the recognition rate from the given input, which is identical to the rank 1 (C)MC value.
The input has a specific format, which is a list of two-element tuples.
Each of the tuples contains the negative and the positive scores for one test item.
To read the lists from score files in 4 or 5 column format, please use the bob.measure.load.cmc_four_column or bob.measure.load.cmc_five_column function.
The recognition rate is defined as the number of test items,
for which the positive score is greater than or equal to all negative scores,
divided by the number of all test items.
If several positive scores for one test item exist, the *highest* score is taken.
"""Calculates the recognition rate from the given input, which is identical
to the rank 1 (C)MC value.
The input has a specific format, which is a list of two-element tuples. Each
of the tuples contains the negative and the positive scores for one test
item. To read the lists from score files in 4 or 5 column format, please use
the :py:func:`xbob.measure.load.cmc_four_column` or
:py:func:`bob.measure.load.cmc_five_column` function.
The recognition rate is defined as the number of test items, for which the
positive score is greater than or equal to all negative scores, divided by
the number of all test items. If several positive scores for one test item
exist, the **highest** score is taken.
"""
correct = 0.
for neg, pos in cmc_scores:
......@@ -90,15 +101,20 @@ def recognition_rate(cmc_scores):
def cmc(cmc_scores):
"""Calculates the cumulative match characteristic (CMC) from the given input.
The input has a specific format, which is a list of two-element tuples.
Each of the tuples contains the negative and the positive scores for one test item.
To read the lists from score files in 4 or 5 column format, please use the bob.measure.load.cmc_four_column or bob.measure.load.cmc_five_column function.
For each test item the probability that the rank r of the positive score is calculated.
The rank is computed as the number of negative scores that are higher than the positive score.
If several positive scores for one test item exist, the *highest* positive score is taken.
The CMC finally computes, how many test items have rank r or higher.
The input has a specific format, which is a list of two-element tuples. Each
of the tuples contains the negative and the positive scores for one test
item. To read the lists from score files in 4 or 5 column format, please use
the :py:func:`xbob.measure.load.cmc_four_column` or
:py:func:`bob.measure.load.cmc_five_column` function.
For each test item the probability that the rank r of the positive score is
calculated. The rank is computed as the number of negative scores that are
higher than the positive score. If several positive scores for one test item
exist, the **highest** positive score is taken. The CMC finally computes how
many test items have rank r or higher.
"""
# compute MC
match_characteristic = numpy.zeros((max([len(neg) for (neg,pos) in cmc_scores])+1,), numpy.int)
for neg, pos in cmc_scores:
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Thu May 16 11:41:49 CEST 2013
# Manuel Guenther <Manuel.Guenther@idiap.ch>
# Thu May 16 11:41:49 CEST 2013
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Measures for calibration"""
......@@ -35,7 +23,7 @@ def cllr(negatives, positives):
def min_cllr(negatives, positives):
"""Computes the 'minimum cost of log likelihood ratio' measure as given in the bosaris toolkit"""
from ..math import pavx
from xbob.math import pavx
# first, sort both scores
neg = sorted(negatives)
......@@ -69,7 +57,6 @@ def min_cllr(negatives, positives):
# ... activate old warnings
numpy.seterr(**old_warn_setup)
llrs = posterior_log_odds - log_prior_odds;
# some weired addition
......@@ -86,4 +73,3 @@ def min_cllr(negatives, positives):
# compute cllr of these new 'optimal' LLR scores
return cllr(new_neg, new_pos)
......@@ -2,6 +2,8 @@
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Mon 23 May 2011 16:23:05 CEST
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
"""A set of utilities to load score files with different formats.
"""
......@@ -73,15 +75,20 @@ def split_four_column(filename):
def cmc_four_column(filename):
"""Loads scores to compute CMC curves from a file in four column format.
The four column file needs to be in the same format as described in the four_column function,
and the "test label" (column 3) has to contain the test/probe file name.
This function returns a list of tuples.
For each probe file, the tuple consists of a list of negative scores and a list of positive scores.
Usually, the list of positive scores should contain only one element, but more are allowed.
The result of this function can directly be passed to, e.g., the bob.measure.cmc function.
The four column file needs to be in the same format as described in the
four_column function, and the "test label" (column 3) has to contain the
test/probe file name.
This function returns a list of tuples. For each probe file, the tuple
consists of a list of negative scores and a list of positive scores.
Usually, the list of positive scores should contain only one element, but
more are allowed.
The result of this function can directly be passed to, e.g., the
:py:func:`xbob.measure.cmc` function.
"""
# read four column list
all_list = four_column(filename)
# extract positives and negatives
......@@ -186,15 +193,20 @@ def split_five_column(filename):
def cmc_five_column(filename):
"""Loads scores to compute CMC curves from a file in five column format.
The four column file needs to be in the same format as described in the five_column function,
and the "test label" (column 4) has to contain the test/probe file name.
This function returns a list of tuples.
For each probe file, the tuple consists of a list of negative scores and a list of positive scores.
Usually, the list of positive scores should contain only one element, but more are allowed.
The four column file needs to be in the same format as described in the
five_column function, and the "test label" (column 4) has to contain the
test/probe file name.
The result of this function can directly be passed to, e.g., the bob.measure.cmc function.
This function returns a list of tuples. For each probe file, the tuple
consists of a list of negative scores and a list of positive scores.
Usually, the list of positive scores should contain only one element, but
more are allowed.
The result of this function can directly be passed to, e.g., the
:py:func:`xbob.measure.cmc` function.
"""
# read four column list
all_list = five_column(filename)
......
......@@ -4,18 +4,6 @@
# Wed May 25 13:27:46 2011 +0200
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This script applies a threshold to score file and reports error rates
"""
......@@ -28,12 +16,14 @@ Examples:
$ %(prog)s --scores=my-scores.txt --threshold=0.5
"""
import sys, os, bob
import os
import sys
from .. import farfrr, load
def apthres(neg, pos, thres):
"""Prints a single output line that contains all info for the threshold"""
far, frr = bob.measure.farfrr(neg, pos, thres)
far, frr = farfrr(neg, pos, thres)
hter = (far + frr)/2.0
ni = neg.shape[0] #number of impostors
......@@ -62,7 +52,7 @@ def get_options(user_input):
parser.add_argument('-t', '--threshold', dest='thres', default=None,
type=float, help="The threshold value to apply", metavar="FLOAT")
parser.add_argument('-p', '--parser', dest="parser", default="4column",
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of xbob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
# This option is not normally shown to the user...
parser.add_argument("--self-test",
......@@ -83,9 +73,9 @@ def get_options(user_input):
#parse the score-parser
if args.parser.lower() in ('4column', '4col'):
args.parser = bob.measure.load.split_four_column
args.parser = load.split_four_column
elif args.parser.lower() in ('5column', '5col'):
args.parser = bob.measure.load.split_five_column
args.parser = load.split_five_column
else: #try an import
if args.parser.find('.') == -1:
parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser)
......
......@@ -4,18 +4,6 @@
# Wed May 25 13:27:46 2011 +0200
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This script runs error analysis on the development and test set scores, in a
four column format:
......@@ -41,20 +29,23 @@ Examples:
$ %(prog)s --no-plot --devel=dev.scores --test=test.scores
"""
import sys, os, bob
import os
import sys
def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
"""Prints a single output line that contains all info for a given criterium"""
from .. import eer_threshold, min_hter_threshold, farfrr
if crit == 'EER':
thres = bob.measure.eer_threshold(dev_neg, dev_pos)
thres = eer_threshold(dev_neg, dev_pos)
else:
thres = bob.measure.min_hter_threshold(dev_neg, dev_pos)
thres = min_hter_threshold(dev_neg, dev_pos)
dev_far, dev_frr = bob.measure.farfrr(dev_neg, dev_pos, thres)
dev_far, dev_frr = farfrr(dev_neg, dev_pos, thres)
dev_hter = (dev_far + dev_frr)/2.0
test_far, test_frr = bob.measure.farfrr(test_neg, test_pos, thres)
test_far, test_frr = farfrr(test_neg, test_pos, thres)
test_hter = (test_far + test_frr)/2.0
print("[Min. criterium: %s] Threshold on Development set: %e" % (crit, thres))
......@@ -93,6 +84,8 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit):
def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
"""Saves ROC, DET and EPC curves on the file pointed out by filename."""
from .. import plot
import matplotlib
if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf')
import matplotlib.pyplot as mpl
......@@ -102,9 +95,9 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
# ROC
fig = mpl.figure()
bob.measure.plot.roc(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3),
plot.roc(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
bob.measure.plot.roc(test_neg, test_pos, npoints, color=(0,0,0),
plot.roc(test_neg, test_pos, npoints, color=(0,0,0),
linestyle='-', label='test')
mpl.axis([0,40,0,40])
mpl.title("ROC Curve")
......@@ -116,11 +109,11 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
# DET
fig = mpl.figure()
bob.measure.plot.det(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3),
plot.det(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3),
linestyle='--', dashes=(6,2), label='development')
bob.measure.plot.det(test_neg, test_pos, npoints, color=(0,0,0),
plot.det(test_neg, test_pos, npoints, color=(0,0,0),
linestyle='-', label='test')
bob.measure.plot.det_axis([0.01, 40, 0.01, 40])
plot.det_axis([0.01, 40, 0.01, 40])
mpl.title("DET Curve")
mpl.xlabel('FRR (%)')
mpl.ylabel('FAR (%)')
......@@ -130,7 +123,7 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
# EPC
fig = mpl.figure()
bob.measure.plot.epc(dev_neg, dev_pos, test_neg, test_pos, npoints,
plot.epc(dev_neg, dev_pos, test_neg, test_pos, npoints,
color=(0,0,0), linestyle='-')
mpl.title('EPC Curve')
mpl.xlabel('Cost')
......@@ -163,7 +156,7 @@ def get_options(user_input):
parser.add_argument('-x', '--no-plot', dest="doplot", default=True,
action='store_false', help="If set, then I'll execute no plotting")
parser.add_argument('-p', '--parser', dest="parser", default="4column",
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of xbob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
# This option is not normally shown to the user...
parser.add_argument("--self-test",
......@@ -186,10 +179,12 @@ def get_options(user_input):
parser.error("you should give a test score set with --test")
#parse the score-parser
from .. import load
if args.parser.lower() in ('4column', '4col'):
args.parser = bob.measure.load.split_four_column
args.parser = load.split_four_column
elif args.parser.lower() in ('5column', '5col'):
args.parser = bob.measure.load.split_five_column
args.parser = load.split_five_column
else: #try an import
if args.parser.find('.') == -1:
parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % arg.parser)
......
......@@ -4,18 +4,6 @@
# Wed May 25 13:27:46 2011 +0200
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This script computes the threshold following a certain minimization criteria
on the given input data."""
......@@ -36,12 +24,15 @@ Examples:
$ %(prog)s --scores=dev.scores --parser=5column
"""
import sys, os, bob
import os
import sys
def apthres(neg, pos, thres):
"""Prints a single output line that contains all info for the threshold"""
far, frr = bob.measure.farfrr(neg, pos, thres)
from .. import farfrr
far, frr = farfrr(neg, pos, thres)
hter = (far + frr)/2.0
ni = neg.shape[0] #number of impostors
......@@ -56,13 +47,15 @@ def apthres(neg, pos, thres):
def calculate(neg, pos, crit, cost):
"""Returns the threshold given a certain criteria"""
from .. import eer_threshold, min_hter_threshold, min_weighted_error_rate_threshold
if crit == 'eer':
return bob.measure.eer_threshold(neg, pos)
return eer_threshold(neg, pos)
elif crit == 'mhter':
return bob.measure.min_hter_threshold(neg, pos)
return min_hter_threshold(neg, pos)
# defaults to the minimum of the weighter error rate
return bob.measure.min_weighted_error_rate_threshold(neg, pos, cost)
return min_weighted_error_rate_threshold(neg, pos, cost)
def get_options(user_input):
"""Parse the program options"""
......@@ -84,7 +77,7 @@ def get_options(user_input):
parser.add_argument('-w', '--cost', dest='cost', default=0.5,
type=float, help="The value w of the cost when minimizing using the minimum weighter error rate (mwer) criterium. This value is ignored for eer or mhter criteria.", metavar="FLOAT")
parser.add_argument('-p', '--parser', dest="parser", default="4column",
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of xbob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION")
# This option is not normally shown to the user...
parser.add_argument("--self-test",
......@@ -100,10 +93,11 @@ def get_options(user_input):
parser.error("cost should lie between 0.0 and 1.0")
#parse the score-parser
from .. import load
if args.parser.lower() in ('4column', '4col'):
args.parser = bob.measure.load.split_four_column
args.parser = load.split_four_column
elif args.parser.lower() in ('5column', '5col'):
args.parser = bob.measure.load.split_five_column
args.parser = load.split_five_column
else: #try an import
if args.parser.find('.') == -1:
parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser)
......
......@@ -20,12 +20,12 @@
"""This script computes and plot a cumulative rank characteristics (CMC) curve
from a score file in four or five column format.
Note: The score file has to contain the exact probe file names as the 3rd (4column) or 4th (5column) column.
Note: The score file has to contain the exact probe file names as the 3rd
(4column) or 4th (5column) column.
"""
import bob, os, sys
import os
import sys
def parse_command_line(command_line_options):
"""Parse the program options"""
......@@ -56,15 +56,18 @@ def parse_command_line(command_line_options):
def main(command_line_options = None):
"""Computes and plots the CMC curve."""
from .. import load, plot, recognition_rate
args = parse_command_line(command_line_options)
# read data
if not os.path.isfile(args.score_file): raise IOError("The given score file does not exist")
# pythonic way: create inline dictionary "{...}", index with desired value "[...]", execute function "(...)"
data = {'4column' : bob.measure.load.cmc_four_column, '5column' : bob.measure.load.cmc_five_column}[args.parser](args.score_file)
data = {'4column' : load.cmc_four_column, '5column' : load.cmc_five_column}[args.parser](args.score_file)
# compute recognition rate
rr = bob.measure.recognition_rate(data)
rr = recognition_rate(data)
print("Recognition rate for score file", args.score_file, "is %3.2f%%" % (rr * 100))
if not args.no_plot:
......@@ -78,7 +81,7 @@ def main(command_line_options = None):
# CMC
fig = mpl.figure()
max_rank = bob.measure.plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
mpl.title("CMC Curve")
if args.log_x_scale:
mpl.xlabel('Rank (log)')
......
......@@ -4,24 +4,12 @@
# Tue Jul 2 14:52:49 CEST 2013
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This script parses through the given directory, collects all results of
verification experiments that are stored in file with the given file name.
It supports the split into development and test set of the data, as well as
verification experiments that are stored in file with the given file name. It
supports the split into development and test set of the data, as well as
ZT-normalized scores.
All result files are parsed and evaluated. For each directory, the following
......@@ -37,16 +25,15 @@ The measure type of the development set can be changed to compute "HTER" or
"FAR" thresholds instead, using the --criterion option.
"""
import sys, os, bob
import os
import sys
#from apport.hookutils import default
def get_args():
"""Parse the program options"""
import argparse
from .. import load
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
......@@ -75,16 +62,16 @@ def get_args():
parser.add_argument('--self-test', action='store_true', help=argparse.SUPPRESS)
parser.add_argument('-p', '--parser', dest="parser", default="4column", metavar="NAME.FUNCTION",
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details")
help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of xbob.measure.load.split_four_column() for details")
# parse arguments
args = parser.parse_args()
# parse the score-parser
if args.parser.lower() in ('4column', '4col'):
args.parser = bob.measure.load.split_four_column
args.parser = load.split_four_column
elif args.parser.lower() in ('5column', '5col'):
args.parser = bob.measure.load.split_five_column
args.parser = load.split_five_column
else: #try an import
if args.parser.find('.') == -1:
parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser)
......@@ -117,23 +104,25 @@ class Result:
self.ztnorm_eval = None
def __calculate__(self, dev_file, eval_file = None):
from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr
dev_neg, dev_pos = self.m_args.parser(dev_file)
# switch which threshold function to use;
# THIS f***ing piece of code really is what python authors propose:
threshold = {
'EER' : bob.measure.eer_threshold,
'HTER' : bob.measure.min_hter_threshold,
'FAR' : bob.measure.far_threshold
'EER' : eer_threshold,
'HTER' : min_hter_threshold,
'FAR' : far_threshold
} [self.m_args.criterion](dev_neg, dev_pos)
# compute far and frr for the given threshold
dev_far, dev_frr = bob.measure.farfrr(dev_neg, dev_pos, threshold)
dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold)
dev_hter = (dev_far + dev_frr)/2.0
if eval_file:
eval_neg, eval_pos = self.m_args.parser(eval_file)
eval_far, eval_frr = bob.measure.farfrr(eval_neg, eval_pos, threshold)
eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold)
eval_hter = (eval_far + eval_frr)/2.0
else:
eval_hter = None
......
This diff is collapsed.
......@@ -4,25 +4,12 @@
# Tue 21 Aug 2012 12:14:43 CEST
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Script tests for bob.measure
"""Script tests for xbob.measure
"""
import os
import unittest
import bob
import nose.tools
import pkg_resources
def F(f):
......@@ -38,53 +25,50 @@ TEST_SCORES_5COL = F('test-5col.txt')
SCORES_4COL_CMC = F('scores-cmc-4col.txt')
SCORES_5COL_CMC = F('scores-cmc-5col.txt')
class MeasureScriptTest(unittest.TestCase):
def test01_compute_perf(self):
# sanity checks
self.assertTrue(os.path.exists(DEV_SCORES))
self.assertTrue(os.path.exists(TEST_SCORES))
def test_compute_perf():
from bob.measure.script.compute_perf import main
cmdline = '--devel=%s --test=%s --self-test' % (DEV_SCORES, TEST_SCORES)
self.assertEqual(main(cmdline.split()), 0)
# sanity checks
assert os.path.exists(DEV_SCORES)
assert os.path.exists(TEST_SCORES)
def test02_eval_threshold(self):
from ..script.compute_perf import main
cmdline = '--devel=%s --test=%s --self-test' % (DEV_SCORES, TEST_SCORES)
nose.tools.eq_(main(cmdline.split()), 0)
# sanity checks
self.assertTrue(os.path.exists(DEV_SCORES))
def test_eval_threshold():
from bob.measure.script.eval_threshold import main
cmdline = '--scores=%s --self-test' % (DEV_SCORES,)
self.assertEqual(main(cmdline.split()), 0)
# sanity checks
assert os.path.exists(DEV_SCORES)
def test03_apply_threshold(self):
from ..script.eval_threshold import main
cmdline = '--scores=%s --self-test' % (DEV_SCORES,)
nose.tools.eq_(main(cmdline.split()), 0)
# sanity checks
self.assertTrue(os.path.exists(TEST_SCORES))
def test_apply_threshold():
from bob.measure.script.apply_threshold import main
cmdline = '--scores=%s --self-test' % (TEST_SCORES,)
self.assertEqual(main(cmdline.split()), 0)
# sanity checks
assert os.path.exists(TEST_SCORES)
def test04_compute_perf_5col(self):
from ..script.apply_threshold import main
cmdline = '--scores=%s --self-test' % (TEST_SCORES,)
nose.tools.eq_(main(cmdline.split()), 0)
# sanity checks
self.assertTrue(os.path.exists(DEV_SCORES_5COL))
self.assertTrue(os.path.exists(TEST_SCORES_5COL))
def test_compute_perf_5col():
from bob.measure.script.compute_perf import main
cmdline = '--devel=%s --test=%s --parser=bob.measure.load.split_five_column --self-test' % (DEV_SCORES_5COL, TEST_SCORES_5COL)
self.assertEqual(main(cmdline.split()), 0)
# sanity checks