Commit 3a64d554 authored by Manuel Günther's avatar Manuel Günther

Merge branch...

Merge branch '27-far-and-frr-thresholds-are-computed-even-when-there-is-no-data-support' into 'master'

Resolve "FAR and FRR thresholds are computed even when there is no data support"

Closes #27

See merge request !43
parents 4281a0c2 cb9cfbb7
Pipeline #14134 passed with stages
in 9 minutes and 8 seconds
...@@ -39,7 +39,7 @@ bob::measure::farfrr(const blitz::Array<double, 1> &negatives, ...@@ -39,7 +39,7 @@ bob::measure::farfrr(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives, const blitz::Array<double, 1> &positives,
double threshold) { double threshold) {
if (std::isnan(threshold)){ if (std::isnan(threshold)){
bob::core::error << "Cannot compute FAR or FRR with threshold NaN"; bob::core::error << "Cannot compute FAR or FRR with threshold NaN.\n";
return std::make_pair(1.,1.); return std::make_pair(1.,1.);
} }
if (!negatives.size()) if (!negatives.size())
...@@ -105,7 +105,7 @@ double bob::measure::eerRocch(const blitz::Array<double, 1> &negatives, ...@@ -105,7 +105,7 @@ double bob::measure::eerRocch(const blitz::Array<double, 1> &negatives,
} }
double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives, double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &, const blitz::Array<double, 1> &positives,
double far_value, bool is_sorted) { double far_value, bool is_sorted) {
// check the parameters are valid // check the parameters are valid
if (far_value < 0. || far_value > 1.) { if (far_value < 0. || far_value > 1.) {
...@@ -119,30 +119,46 @@ double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives, ...@@ -119,30 +119,46 @@ double bob::measure::farThreshold(const blitz::Array<double, 1> &negatives,
"the number of negative scores must be at least 2"); "the number of negative scores must be at least 2");
} }
// sort the array, if necessary // sort the negatives array, if necessary, and keep it in the scores variable
blitz::Array<double, 1> neg; blitz::Array<double, 1> scores;
sort(negatives, neg, is_sorted); sort(negatives, scores, is_sorted);
// compute position of the threshold double epsilon = std::numeric_limits<double>::epsilon();
double crr = 1. - far_value; // (Correct Rejection Rate; = 1 - FAR) // handle special case of far == 1 without any iterating
double crr_index = std::max(crr * neg.extent(0) - 1., 0.); if (far_value >= 1 - epsilon)
// compute the index above the current CRR value return nexttoward(scores(0), scores(0)-1);
int index = (int)std::ceil(crr_index);
// Reverse negatives so the end is the start. This way the code below will be
// increase the threshold when we have several negatives with the same score // very similar to the implementation in the frrThreshold function. The
while (index < neg.extent(0)-1 && neg(index) == neg(index+1)) // implementations are not exactly the same though.
++index; scores.reverseSelf(0);
// Move towards the end of array changing the threshold until we pass the
if (index < neg.extent(0)-1){ // desired FAR value. Starting with a threshold that corresponds to FAR == 0.
// return the threshold that is just above the desired FAR int total_count = scores.extent(0);
return neg(index); int current_position = 0;
} else { // since the comparison is `if score >= threshold then accept as genuine`, we
// We cannot reach the desired threshold, as we have too many identical lowest scores, or the number of scores is too low // can choose the largest score value + eps as the threshold so that we can
return std::numeric_limits<double>::quiet_NaN(); // get for 0% FAR.
double valid_threshold = nexttoward(scores(current_position), scores(current_position)+1);
double current_threshold;
double future_far;
while (current_position < total_count) {
current_threshold = scores(current_position);
// keep iterating if values are repeated
while (current_position < total_count-1 && scores(current_position+1) == current_threshold)
current_position++;
// All the scores up to the current position and including the current
// position will be accepted falsely.
future_far = (double)(current_position+1) / (double)total_count;
if (future_far > far_value)
break;
valid_threshold = current_threshold;
current_position++;
} }
return valid_threshold;
} }
double bob::measure::frrThreshold(const blitz::Array<double, 1> &, double bob::measure::frrThreshold(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives, const blitz::Array<double, 1> &positives,
double frr_value, bool is_sorted) { double frr_value, bool is_sorted) {
...@@ -158,27 +174,38 @@ double bob::measure::frrThreshold(const blitz::Array<double, 1> &, ...@@ -158,27 +174,38 @@ double bob::measure::frrThreshold(const blitz::Array<double, 1> &,
"the number of positive scores must be at least 2"); "the number of positive scores must be at least 2");
} }
// sort positive scores descendantly, if necessary // sort the positives array, if necessary, and keep it in the scores variable
blitz::Array<double, 1> pos; blitz::Array<double, 1> scores;
sort(positives, pos, is_sorted); sort(positives, scores, is_sorted);
// compute position of the threshold double epsilon = std::numeric_limits<double>::epsilon();
double frr_index = std::max(frr_value * pos.extent(0) - 1., 0.); // handle special case of frr == 1 without any iterating
// compute the index below the current FAR value if (frr_value >= 1 - epsilon)
int index = (int)std::ceil(frr_index); return nexttoward(scores(scores.extent(0)-1), scores(scores.extent(0)-1)+1);
// lower the threshold when several positives have the same score // Move towards the end of array changing the threshold until we pass the
while (index && pos(index) == pos(index-1)) // desired FRR value. Starting with a threshold that corresponds to FRR == 0.
--index; int total_count = scores.extent(0);
int current_position = 0;
if (index){ // since the comparison is `if score >= threshold then accept as genuine`, we
// return the FRR threshold that is just above the desired FRR // can use the smallest positive score as the threshold for 0% FRR.
// We have to add a little noise to since the FRR calculation excludes the threshold double valid_threshold = scores(current_position);
return pos(index) + 1e-8 * pos(index); double current_threshold;
} else { double future_frr;
// We cannot reach the desired threshold, as we have too many identical highest scores while (current_position < total_count) {
return std::numeric_limits<double>::quiet_NaN(); current_threshold = scores(current_position);
// keep iterating if values are repeated
while (current_position < total_count-1 && scores(current_position+1) == current_threshold)
current_position++;
// All the scores up to the current_position but not including
// current_position will be rejected falsely.
future_frr = (double)current_position / (double)total_count;
if (future_frr > frr_value)
break;
valid_threshold = current_threshold;
current_position++;
} }
return valid_threshold;
} }
/** /**
......
...@@ -713,7 +713,7 @@ static PyObject *precision_recall_curve(PyObject *, PyObject *args, ...@@ -713,7 +713,7 @@ static PyObject *precision_recall_curve(PyObject *, PyObject *args,
static auto far_threshold_doc = static auto far_threshold_doc =
bob::extension::FunctionDoc( bob::extension::FunctionDoc(
"far_threshold", "Computes the threshold such that the real FAR is " "far_threshold", "Computes the threshold such that the real FAR is "
"**at least** the requested ``far_value`` if possible", "**at most** the requested ``far_value`` if possible",
"If no such threshold can be computed, ``NaN`` is returned. It is " "If no such threshold can be computed, ``NaN`` is returned. It is "
"impossible to compute the threshold, when too few non-identical " "impossible to compute the threshold, when too few non-identical "
"highest scores exist, so that the desired ``far_value`` cannot be " "highest scores exist, so that the desired ``far_value`` cannot be "
...@@ -742,7 +742,7 @@ static auto far_threshold_doc = ...@@ -742,7 +742,7 @@ static auto far_threshold_doc =
"will require more memory") "will require more memory")
.add_return( .add_return(
"threshold", "float", "threshold", "float",
"The threshold such that the real FAR is at least ``far_value``"); "The threshold such that the real FAR is at most ``far_value``");
static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) { static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) {
BOB_TRY BOB_TRY
static char **kwlist = far_threshold_doc.kwlist(); static char **kwlist = far_threshold_doc.kwlist();
...@@ -773,7 +773,7 @@ static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) { ...@@ -773,7 +773,7 @@ static PyObject *far_threshold(PyObject *, PyObject *args, PyObject *kwds) {
static auto frr_threshold_doc = static auto frr_threshold_doc =
bob::extension::FunctionDoc( bob::extension::FunctionDoc(
"frr_threshold", "Computes the threshold such that the real FRR is " "frr_threshold", "Computes the threshold such that the real FRR is "
"**at least** the requested ``frr_value`` if possible", "**at most** the requested ``frr_value`` if possible",
"If no such threshold can be computed, ``NaN`` is returned. It is " "If no such threshold can be computed, ``NaN`` is returned. It is "
"impossible to compute the threshold, when too few non-identical " "impossible to compute the threshold, when too few non-identical "
"lowest scores exist, so that the desired ``frr_value`` cannot be " "lowest scores exist, so that the desired ``frr_value`` cannot be "
...@@ -802,7 +802,7 @@ static auto frr_threshold_doc = ...@@ -802,7 +802,7 @@ static auto frr_threshold_doc =
"will require more memory") "will require more memory")
.add_return( .add_return(
"threshold", "float", "threshold", "float",
"The threshold such that the real FRR is at least ``frr_value``"); "The threshold such that the real FRR is at most ``frr_value``");
static PyObject *frr_threshold(PyObject *, PyObject *args, PyObject *kwds) { static PyObject *frr_threshold(PyObject *, PyObject *args, PyObject *kwds) {
BOB_TRY BOB_TRY
char **kwlist = frr_threshold_doc.kwlist(); char **kwlist = frr_threshold_doc.kwlist();
......
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
"""Basic tests for the error measuring system of bob """Basic tests for the error measuring system of bob
""" """
from __future__ import division
import os import os
import numpy import numpy
import nose.tools import nose.tools
...@@ -84,43 +84,59 @@ def test_basic_ratios(): ...@@ -84,43 +84,59 @@ def test_basic_ratios():
nose.tools.eq_(f_score_, 1.0) nose.tools.eq_(f_score_, 1.0)
def test_nan_for_uncomputable_thresholds(): def test_for_uncomputable_thresholds():
# in some cases, we cannot compute an FAR or FRR threshold, e.g., when we have too little data or too many equal scores # in some cases, we cannot compute an FAR or FRR threshold, e.g., when we
# in these cases, the methods should return NaN # have too little data or too many equal scores in these cases, the methods
# should return a threshold which a supports a lower value.
from . import far_threshold, frr_threshold from . import far_threshold, frr_threshold
# case 1: several scores are identical # case 1: several scores are identical
positives = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5] pos = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
negatives = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0] neg = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0]
# test that reasonable thresholds for reachable data points are provided # test that reasonable thresholds for reachable data points are provided
assert far_threshold(negatives, positives, 0.5) == 0.9 threshold = far_threshold(neg, pos, 0.5)
assert numpy.isclose(frr_threshold(negatives, positives, 0.5), 0.1) assert threshold == 1.0, threshold
threshold = frr_threshold(neg, pos, 0.5)
assert numpy.isclose(threshold, 0.1), threshold
assert math.isnan(far_threshold(negatives, positives, 0.4)) threshold = far_threshold(neg, pos, 0.4)
assert math.isnan(frr_threshold(negatives, positives, 0.4)) assert threshold > neg[-1], threshold
threshold = frr_threshold(neg, pos, 0.4)
assert threshold >= pos[0], threshold
# test the same with even number of scores # test the same with even number of scores
positives = [0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5] pos = [0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
negatives = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0] neg = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0]
assert far_threshold(negatives, positives, 0.5) == 0.9
assert numpy.isclose(frr_threshold(negatives, positives, 0.51), 0.1)
assert math.isnan(far_threshold(negatives, positives, 0.49))
assert math.isnan(frr_threshold(negatives, positives, 0.5))
threshold = far_threshold(neg, pos, 0.5)
assert threshold == 1.0, threshold
assert numpy.isclose(frr_threshold(neg, pos, 0.51), 0.1)
threshold = far_threshold(neg, pos, 0.49)
assert threshold > neg[-1], threshold
threshold = frr_threshold(neg, pos, 0.49)
assert threshold >= pos[0], threshold
# case 2: too few scores for the desired threshold # case 2: too few scores for the desired threshold
positives = numpy.arange(10.) pos = numpy.array(range(10), dtype=float)
negatives = numpy.arange(10.) neg = numpy.array(range(10), dtype=float)
assert math.isnan(far_threshold(negatives, positives, 0.09)) threshold = far_threshold(neg, pos, 0.09)
assert math.isnan(frr_threshold(negatives, positives, 0.09)) assert threshold > neg[-1], threshold
# there is no limit above; the threshold will just be the largest possible value threshold = frr_threshold(neg, pos, 0.09)
assert far_threshold(negatives, positives, 0.11) == 8. assert threshold >= pos[0], threshold
assert far_threshold(negatives, positives, 0.91) == 0. # there is no limit above; the threshold will just be the largest possible
assert numpy.isclose(frr_threshold(negatives, positives, 0.11), 1.) # value
assert numpy.isclose(frr_threshold(negatives, positives, 0.91), 9.) threshold = far_threshold(neg, pos, 0.11)
assert threshold == 9., threshold
threshold = far_threshold(neg, pos, 0.91)
assert threshold == 1., threshold
threshold = far_threshold(neg, pos, 1)
assert threshold <= 0., threshold
threshold = frr_threshold(neg, pos, 0.11)
assert numpy.isclose(threshold, 1.), threshold
threshold = frr_threshold(neg, pos, 0.91)
assert numpy.isclose(threshold, 9.), threshold
def test_indexing(): def test_indexing():
...@@ -151,6 +167,27 @@ def test_indexing(): ...@@ -151,6 +167,27 @@ def test_indexing():
assert correctly_classified_negatives(negatives, 3).all() assert correctly_classified_negatives(negatives, 3).all()
def test_obvious_thresholds():
from . import far_threshold, frr_threshold, farfrr
M = 10
neg = numpy.arange(M, dtype=float)
pos = numpy.arange(M, 2 * M, dtype=float)
for far, frr in zip(numpy.arange(0, 2 * M + 1, dtype=float) / M / 2,
numpy.arange(0, 2 * M + 1, dtype=float) / M / 2):
far, expected_far = round(far, 2), math.floor(far * 10) / 10
frr, expected_frr = round(frr, 2), math.floor(frr * 10) / 10
calculated_far_threshold = far_threshold(neg, pos, far)
pred_far, _ = farfrr(neg, pos, calculated_far_threshold)
calculated_frr_threshold = frr_threshold(neg, pos, frr)
_, pred_frr = farfrr(neg, pos, calculated_frr_threshold)
assert pred_far <= far, (pred_far, far, calculated_far_threshold)
assert pred_far == expected_far, (pred_far, far, calculated_far_threshold)
assert pred_frr <= frr, (pred_frr, frr, calculated_frr_threshold)
assert pred_frr == expected_frr, (pred_frr, frr, calculated_frr_threshold)
def test_thresholding(): def test_thresholding():
from . import eer_threshold, far_threshold, frr_threshold, farfrr, \ from . import eer_threshold, far_threshold, frr_threshold, farfrr, \
...@@ -186,12 +223,12 @@ def test_thresholding(): ...@@ -186,12 +223,12 @@ def test_thresholding():
far = farfrr(negatives, positives, threshold_far)[0] far = farfrr(negatives, positives, threshold_far)[0]
frr = farfrr(negatives, positives, threshold_frr)[1] frr = farfrr(negatives, positives, threshold_frr)[1]
if not math.isnan(threshold_far): if not math.isnan(threshold_far):
assert far + 1e-7 > t, (far,t) assert far <= t, (far, t)
assert far - t <= 0.1 assert t - far <= 0.1
if not math.isnan(threshold_frr): if not math.isnan(threshold_frr):
assert frr + 1e-7 > t, (frr,t) assert frr <= t, (frr, t)
# test that the values are at least somewhere in the range # test that the values are at least somewhere in the range
assert frr - t <= 0.1 assert t - frr <= 0.1
# If the set is separable, the calculation of the threshold is a little bit # If the set is separable, the calculation of the threshold is a little bit
# trickier, as you have no points in the middle of the range to compare # trickier, as you have no points in the middle of the range to compare
......
2.4.2b0 2.5.0b0
\ No newline at end of file \ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment