Commit 3a64d554 authored by Manuel Günther's avatar Manuel Günther

Merge branch...

Merge branch '27-far-and-frr-thresholds-are-computed-even-when-there-is-no-data-support' into 'master'

Resolve "FAR and FRR thresholds are computed even when there is no data support"

Closes #27

See merge request !43
parents 4281a0c2 cb9cfbb7
Pipeline #14134 passed with stages
in 9 minutes and 8 seconds
......@@ -39,7 +39,7 @@ bob::measure::farfrr(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives,
double threshold) {
if (std::isnan(threshold)){
bob::core::error << "Cannot compute FAR or FRR with threshold NaN";
bob::core::error << "Cannot compute FAR or FRR with threshold NaN.\n";
return std::make_pair(1.,1.);
}
if (!negatives.size())
......@@ -105,7 +105,7 @@ double bob::measure::eerRocch(const blitz::Array<double, 1> &negatives,
}
double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &,
const blitz::Array<double, 1> &positives,
double far_value, bool is_sorted) {
// check the parameters are valid
if (far_value < 0. || far_value > 1.) {
......@@ -119,30 +119,46 @@ double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives,
"the number of negative scores must be at least 2");
}
// sort the array, if necessary
blitz::Array<double, 1> neg;
sort(negatives, neg, is_sorted);
// compute position of the threshold
double crr = 1. - far_value; // (Correct Rejection Rate; = 1 - FAR)
double crr_index = std::max(crr * neg.extent(0) - 1., 0.);
// compute the index above the current CRR value
int index = (int)std::ceil(crr_index);
// increase the threshold when we have several negatives with the same score
while (index < neg.extent(0)-1 && neg(index) == neg(index+1))
++index;
if (index < neg.extent(0)-1){
// return the threshold that is just above the desired FAR
return neg(index);
} else {
// We cannot reach the desired threshold, as we have too many identical lowest scores, or the number of scores is too low
return std::numeric_limits<double>::quiet_NaN();
// sort the negatives array, if necessary, and keep it in the scores variable
blitz::Array<double, 1> scores;
sort(negatives, scores, is_sorted);
double epsilon = std::numeric_limits<double>::epsilon();
// handle special case of far == 1 without any iterating
if (far_value >= 1 - epsilon)
return nexttoward(scores(0), scores(0)-1);
// Reverse negatives so the end is the start. This way the code below will be
// very similar to the implementation in the frrThreshold function. The
// implementations are not exactly the same though.
scores.reverseSelf(0);
// Move towards the end of array changing the threshold until we pass the
// desired FAR value. Starting with a threshold that corresponds to FAR == 0.
int total_count = scores.extent(0);
int current_position = 0;
// since the comparison is `if score >= threshold then accept as genuine`, we
// can choose the largest score value + eps as the threshold so that we can
// get for 0% FAR.
double valid_threshold = nexttoward(scores(current_position), scores(current_position)+1);
double current_threshold;
double future_far;
while (current_position < total_count) {
current_threshold = scores(current_position);
// keep iterating if values are repeated
while (current_position < total_count-1 && scores(current_position+1) == current_threshold)
current_position++;
// All the scores up to the current position and including the current
// position will be accepted falsely.
future_far = (double)(current_position+1) / (double)total_count;
if (future_far > far_value)
break;
valid_threshold = current_threshold;
current_position++;
}
return valid_threshold;
}
double bob::measure::frrThreshold(const blitz::Array<double, 1> &,
double bob::measure::frrThreshold(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives,
double frr_value, bool is_sorted) {
......@@ -158,27 +174,38 @@ double bob::measure::frrThreshold(const blitz::Array<double, 1> &,
"the number of positive scores must be at least 2");
}
// sort positive scores descendantly, if necessary
blitz::Array<double, 1> pos;
sort(positives, pos, is_sorted);
// compute position of the threshold
double frr_index = std::max(frr_value * pos.extent(0) - 1., 0.);
// compute the index below the current FAR value
int index = (int)std::ceil(frr_index);
// lower the threshold when several positives have the same score
while (index && pos(index) == pos(index-1))
--index;
if (index){
// return the FRR threshold that is just above the desired FRR
// We have to add a little noise to since the FRR calculation excludes the threshold
return pos(index) + 1e-8 * pos(index);
} else {
// We cannot reach the desired threshold, as we have too many identical highest scores
return std::numeric_limits<double>::quiet_NaN();
// sort the positives array, if necessary, and keep it in the scores variable
blitz::Array<double, 1> scores;
sort(positives, scores, is_sorted);
double epsilon = std::numeric_limits<double>::epsilon();
// handle special case of frr == 1 without any iterating
if (frr_value >= 1 - epsilon)
return nexttoward(scores(scores.extent(0)-1), scores(scores.extent(0)-1)+1);
// Move towards the end of array changing the threshold until we pass the
// desired FRR value. Starting with a threshold that corresponds to FRR == 0.
int total_count = scores.extent(0);
int current_position = 0;
// since the comparison is `if score >= threshold then accept as genuine`, we
// can use the smallest positive score as the threshold for 0% FRR.
double valid_threshold = scores(current_position);
double current_threshold;
double future_frr;
while (current_position < total_count) {
current_threshold = scores(current_position);
// keep iterating if values are repeated
while (current_position < total_count-1 && scores(current_position+1) == current_threshold)
current_position++;
// All the scores up to the current_position but not including
// current_position will be rejected falsely.
future_frr = (double)current_position / (double)total_count;
if (future_frr > frr_value)
break;
valid_threshold = current_threshold;
current_position++;
}
return valid_threshold;
}
/**
......
......@@ -713,7 +713,7 @@ static PyObject *precision_recall_curve(PyObject *, PyObject *args,
static auto far_threshold_doc =
bob::extension::FunctionDoc(
"far_threshold", "Computes the threshold such that the real FAR is "
"**at least** the requested ``far_value`` if possible",
"**at most** the requested ``far_value`` if possible",
"If no such threshold can be computed, ``NaN`` is returned. It is "
"impossible to compute the threshold, when too few non-identical "
"highest scores exist, so that the desired ``far_value`` cannot be "
......@@ -742,7 +742,7 @@ static auto far_threshold_doc =
"will require more memory")
.add_return(
"threshold", "float",
"The threshold such that the real FAR is at least ``far_value``");
"The threshold such that the real FAR is at most ``far_value``");
static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) {
BOB_TRY
static char **kwlist = far_threshold_doc.kwlist();
......@@ -773,7 +773,7 @@ static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) {
static auto frr_threshold_doc =
bob::extension::FunctionDoc(
"frr_threshold", "Computes the threshold such that the real FRR is "
"**at least** the requested ``frr_value`` if possible",
"**at most** the requested ``frr_value`` if possible",
"If no such threshold can be computed, ``NaN`` is returned. It is "
"impossible to compute the threshold, when too few non-identical "
"lowest scores exist, so that the desired ``frr_value`` cannot be "
......@@ -802,7 +802,7 @@ static auto frr_threshold_doc =
"will require more memory")
.add_return(
"threshold", "float",
"The threshold such that the real FRR is at least ``frr_value``");
"The threshold such that the real FRR is at most ``frr_value``");
static PyObject *frr_threshold(PyObject *, PyObject *args, PyObject *kwds) {
BOB_TRY
char **kwlist = frr_threshold_doc.kwlist();
......
......@@ -7,7 +7,7 @@
"""Basic tests for the error measuring system of bob
"""
from __future__ import division
import os
import numpy
import nose.tools
......@@ -84,43 +84,59 @@ def test_basic_ratios():
nose.tools.eq_(f_score_, 1.0)
def test_nan_for_uncomputable_thresholds():
# in some cases, we cannot compute an FAR or FRR threshold, e.g., when we have too little data or too many equal scores
# in these cases, the methods should return NaN
def test_for_uncomputable_thresholds():
# in some cases, we cannot compute an FAR or FRR threshold, e.g., when we
# have too little data or too many equal scores in these cases, the methods
# should return a threshold which a supports a lower value.
from . import far_threshold, frr_threshold
# case 1: several scores are identical
positives = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
negatives = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0]
pos = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
neg = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0]
# test that reasonable thresholds for reachable data points are provided
assert far_threshold(negatives, positives, 0.5) == 0.9
assert numpy.isclose(frr_threshold(negatives, positives, 0.5), 0.1)
threshold = far_threshold(neg, pos, 0.5)
assert threshold == 1.0, threshold
threshold = frr_threshold(neg, pos, 0.5)
assert numpy.isclose(threshold, 0.1), threshold
assert math.isnan(far_threshold(negatives, positives, 0.4))
assert math.isnan(frr_threshold(negatives, positives, 0.4))
threshold = far_threshold(neg, pos, 0.4)
assert threshold > neg[-1], threshold
threshold = frr_threshold(neg, pos, 0.4)
assert threshold >= pos[0], threshold
# test the same with even number of scores
positives = [0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
negatives = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0]
assert far_threshold(negatives, positives, 0.5) == 0.9
assert numpy.isclose(frr_threshold(negatives, positives, 0.51), 0.1)
assert math.isnan(far_threshold(negatives, positives, 0.49))
assert math.isnan(frr_threshold(negatives, positives, 0.5))
pos = [0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
neg = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0]
threshold = far_threshold(neg, pos, 0.5)
assert threshold == 1.0, threshold
assert numpy.isclose(frr_threshold(neg, pos, 0.51), 0.1)
threshold = far_threshold(neg, pos, 0.49)
assert threshold > neg[-1], threshold
threshold = frr_threshold(neg, pos, 0.49)
assert threshold >= pos[0], threshold
# case 2: too few scores for the desired threshold
positives = numpy.arange(10.)
negatives = numpy.arange(10.)
assert math.isnan(far_threshold(negatives, positives, 0.09))
assert math.isnan(frr_threshold(negatives, positives, 0.09))
# there is no limit above; the threshold will just be the largest possible value
assert far_threshold(negatives, positives, 0.11) == 8.
assert far_threshold(negatives, positives, 0.91) == 0.
assert numpy.isclose(frr_threshold(negatives, positives, 0.11), 1.)
assert numpy.isclose(frr_threshold(negatives, positives, 0.91), 9.)
pos = numpy.array(range(10), dtype=float)
neg = numpy.array(range(10), dtype=float)
threshold = far_threshold(neg, pos, 0.09)
assert threshold > neg[-1], threshold
threshold = frr_threshold(neg, pos, 0.09)
assert threshold >= pos[0], threshold
# there is no limit above; the threshold will just be the largest possible
# value
threshold = far_threshold(neg, pos, 0.11)
assert threshold == 9., threshold
threshold = far_threshold(neg, pos, 0.91)
assert threshold == 1., threshold
threshold = far_threshold(neg, pos, 1)
assert threshold <= 0., threshold
threshold = frr_threshold(neg, pos, 0.11)
assert numpy.isclose(threshold, 1.), threshold
threshold = frr_threshold(neg, pos, 0.91)
assert numpy.isclose(threshold, 9.), threshold
def test_indexing():
......@@ -151,6 +167,27 @@ def test_indexing():
assert correctly_classified_negatives(negatives, 3).all()
def test_obvious_thresholds():
from . import far_threshold, frr_threshold, farfrr
M = 10
neg = numpy.arange(M, dtype=float)
pos = numpy.arange(M, 2 * M, dtype=float)
for far, frr in zip(numpy.arange(0, 2 * M + 1, dtype=float) / M / 2,
numpy.arange(0, 2 * M + 1, dtype=float) / M / 2):
far, expected_far = round(far, 2), math.floor(far * 10) / 10
frr, expected_frr = round(frr, 2), math.floor(frr * 10) / 10
calculated_far_threshold = far_threshold(neg, pos, far)
pred_far, _ = farfrr(neg, pos, calculated_far_threshold)
calculated_frr_threshold = frr_threshold(neg, pos, frr)
_, pred_frr = farfrr(neg, pos, calculated_frr_threshold)
assert pred_far <= far, (pred_far, far, calculated_far_threshold)
assert pred_far == expected_far, (pred_far, far, calculated_far_threshold)
assert pred_frr <= frr, (pred_frr, frr, calculated_frr_threshold)
assert pred_frr == expected_frr, (pred_frr, frr, calculated_frr_threshold)
def test_thresholding():
from . import eer_threshold, far_threshold, frr_threshold, farfrr, \
......@@ -186,12 +223,12 @@ def test_thresholding():
far = farfrr(negatives, positives, threshold_far)[0]
frr = farfrr(negatives, positives, threshold_frr)[1]
if not math.isnan(threshold_far):
assert far + 1e-7 > t, (far,t)
assert far - t <= 0.1
assert far <= t, (far, t)
assert t - far <= 0.1
if not math.isnan(threshold_frr):
assert frr + 1e-7 > t, (frr,t)
assert frr <= t, (frr, t)
# test that the values are at least somewhere in the range
assert frr - t <= 0.1
assert t - frr <= 0.1
# If the set is separable, the calculation of the threshold is a little bit
# trickier, as you have no points in the middle of the range to compare
......
2.4.2b0
\ No newline at end of file
2.5.0b0
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment