Commit c85097b5 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Resolve merge conflicts

parents d3d98729 a2ebcf46
Pipeline #18429 passed with stage
in 21 minutes and 54 seconds
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import numpy
from .utils import confidence_for_indicator_variable
def test_confidence_interval():
def assert_confidence(x, n, expected_lower, expected_upper):
lower, upper = confidence_for_indicator_variable(x, n)
assert numpy.allclose(lower, expected_lower)
assert numpy.allclose(upper, expected_upper)
assert_confidence(1, 2, 0.01257911709342505, 0.98742088290657493)
assert_confidence(10, 10, 0.69150289218123917, 1)
assert_confidence(0, 10, 0, 0.30849710781876077)
''' utility functions for bob.measure '''
import numpy
import scipy.stats
import click
import bob.core
......@@ -109,3 +110,32 @@ def get_colors(n):
return ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728',
'#9467bd', '#8c564b', '#e377c2', '#7f7f7f',
'#bcbd22', '#17becf']
def confidence_for_indicator_variable(x, n, alpha=0.05):
'''Calculates the confidence interval for proportion estimates
The Clopper-Pearson interval method is used for estimating the confidence
intervals.
Parameters
----------
x : int
The number of successes.
n : int
The number of trials.
alpha : :obj:`float`, optional
The 1-confidence value that you want. For example, alpha should be 0.05
to obtain 95% confidence intervals.
Returns
-------
(:obj:`float`, :obj:`float`)
a tuple of (lower_bound, upper_bound) which
shows the limit of your success rate: lower_bound < x/n < upper_bound
'''
lower_bound = scipy.stats.beta.ppf(alpha / 2.0, x, n - x + 1)
upper_bound = scipy.stats.beta.ppf(1 - alpha / 2.0, x + 1, n - x)
if numpy.isnan(lower_bound):
lower_bound = 0
if numpy.isnan(upper_bound):
upper_bound = 1
return (lower_bound, upper_bound)
......@@ -35,9 +35,11 @@ requirements:
- numpy {{ numpy }}
- docopt {{ docopt }}
- tabulate {{ tabulate }}
- scipy {{ scipy }}
run:
- python
- setuptools
- scipy
- matplotlib
- boost
- {{ pin_compatible('numpy') }}
......
......@@ -210,6 +210,40 @@ Both functions require that at least one probe item exists, which has no accordi
>>> dir = bob.measure.detection_identification_rate(rr_scores, threshold = 0, rank=1)
>>> far = bob.measure.false_alarm_rate(rr_scores, threshold = 0)
Confidence interval
-------------------
A confidence interval for parameter :math:`x` consists of a lower
estimate :math:`L`, and an upper estimate :math:`U`, such that the probability
of the true value being within the interval estimate is equal to :math:`\alpha`.
For example, a 95% confidence interval (i.e. :math:`\alpha = 0.95`) for a
parameter :math:`x` is given by :math:`[L, U]` such that
.. math:: Prob(x∈[L,U]) = 95%
The smaller the test size, the wider the confidence
interval will be, and the greater :math:`\alpha`, the smaller the confidence interval
will be.
`The Clopper-Pearson interval`_, a common method for calculating
confidence intervals, is function of the number of success, the number of trials
and confidence
value :math:`\alpha` is used as :py:func:`bob.measure.utils.confidence_for_indicator_variable`.
It is based on the cumulative probabilities of the binomial distribution. This
method is quite conservative, meaning that the true coverage rate of a 95%
Clopper–Pearson interval may be well above 95%.
For example, we want to evaluate the reliability of a system to
identify registered persons. Let's say that among 10,000 accepted
transactions, 9856 are true matches. The 95% confidence interval for true match
rate is then:
.. doctest:: python
>>> bob.measure.utils.confidence_for_indicator_variable(9856, 10000)
(0.98306835053282549, 0.98784270928084694)
meaning there is a 95% probability that the true match rate is inside :math:`[0.983,
0.988]`.
Plotting
--------
......@@ -570,4 +604,4 @@ will output metrics and plots for the two experiments (dev and test pairs) in
.. _`The Expected Performance Curve`: http://publications.idiap.ch/downloads/reports/2005/bengio_2005_icml.pdf
.. _`The DET curve in assessment of detection task performance`: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf
.. _openbr: http://openbiometrics.org
.. _`The Clopper-Pearson interval`: https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Clopper-Pearson_interval
......@@ -76,6 +76,13 @@ Generic
bob.measure.rmse
bob.measure.get_config
Confidence interval
-------------------
.. autosummary::
bob.measure.utils.confidence_for_indicator_variable
Calibration
-----------
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment