Merge pull request #12 from bioidiap/DIR

Incorporate Detection and Identification Rate and Curve
parents cf70b274 3991aee9
This diff is collapsed.
......@@ -10,6 +10,9 @@ import numpy
import tarfile
import os
import logging
logger = logging.getLogger('bob.measure')
def open_file(filename, mode='rt'):
"""open_file(filename) -> file_like
......@@ -121,7 +124,6 @@ def split_four_column(filename):
def cmc_four_column(filename):
"""cmc_four_column(filename) -> cmc_scores
Loads scores to compute CMC curves from a file in four column format.
The four column file needs to be in the same format as described in :py:func:`four_column`,
......@@ -132,7 +134,7 @@ def cmc_four_column(filename):
Usually, the list of positive scores should contain only one element, but more are allowed.
The result of this function can directly be passed to, e.g., the :py:func:`bob.measure.cmc` function.
**Parameters:**
``filename`` : str or file-like
......@@ -141,48 +143,29 @@ def cmc_four_column(filename):
**Returns:**
``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database
``cmc_scores`` : [(negatives, positives)]
A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database.
Both ``negatives`` and ``positives`` can be either an 1D :py:class:`numpy.ndarray` of type ``float``, or ``None``.
"""
# extract positives and negatives
pos_dict = {}
neg_dict = {}
# read four column list
for (client_id, probe_id, probe_name, score_str) in four_column(filename):
try:
score = float(score_str)
# check in which dict we have to put the score
if client_id == probe_id:
correct_dict = pos_dict
else:
correct_dict = neg_dict
# append score
if probe_name in correct_dict:
correct_dict[probe_name].append(score)
else:
correct_dict[probe_name] = [score]
except:
raise SyntaxError("Cannot convert score '%s' to float" % score_str)
# convert to lists of tuples of ndarrays
retval = []
import logging
logger = logging.getLogger('bob')
for probe_name in sorted(pos_dict.keys()):
if probe_name in neg_dict:
retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
# read four column list
for (client_id, probe_id, probe_name, score) in four_column(filename):
# check in which dict we have to put the score
correct_dict = pos_dict if client_id == probe_id else neg_dict
# append score
if probe_name in correct_dict:
correct_dict[probe_name].append(score)
else:
logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
correct_dict[probe_name] = [score]
#test if there are probes for which only negatives exist
for probe_name in sorted(neg_dict.keys()):
if not probe_name in pos_dict.keys():
logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
# convert that into the desired format
return _convert_cmc_scores(neg_dict, pos_dict)
return retval
def five_column(filename):
"""five_column(filename) -> claimed_id, model_label, real_id, test_label, score
......@@ -259,7 +242,7 @@ def split_five_column(filename):
def cmc_five_column(filename):
"""cmc_four_column(filename) -> cmc_scores
Loads scores to compute CMC curves from a file in five column format.
The four column file needs to be in the same format as described in :py:func:`five_column`,
and the ``test_label`` (column 4) has to contain the test/probe file name or a probe id.
......@@ -286,35 +269,19 @@ def cmc_five_column(filename):
# read four column list
for (client_id, _, probe_id, probe_name, score) in five_column(filename):
# check in which dict we have to put the score
if client_id == probe_id:
correct_dict = pos_dict
else:
correct_dict = neg_dict
correct_dict = pos_dict if client_id == probe_id else neg_dict
# append score
if probe_name in correct_dict:
correct_dict[probe_name].append(score)
else:
correct_dict[probe_name] = [score]
# convert to lists of tuples of ndarrays
retval = []
import logging
logger = logging.getLogger('bob')
for probe_name in sorted(pos_dict.keys()):
if probe_name in neg_dict:
retval.append((numpy.array(neg_dict[probe_name], numpy.float64), numpy.array(pos_dict[probe_name], numpy.float64)))
else:
logger.warn('For probe name "%s" there are only positive scores. This probe name is ignored.' % probe_name)
# test if there are probes for which only negatives exist
for probe_name in sorted(neg_dict.keys()):
if not probe_name in pos_dict.keys():
logger.warn('For probe name "%s" there are only negative scores. This probe name is ignored.' % probe_name)
# convert that into the desired format
return _convert_cmc_scores(neg_dict, pos_dict)
return retval
def load_score(filename, ncolumns=None):
def load_score(filename, ncolumns = 4):
"""Load scores using numpy.loadtxt and return the data as a numpy array.
**Parameters:**
......@@ -333,11 +300,8 @@ def load_score(filename, ncolumns=None):
'claimed_id', 'real_id', 'test_label', and ['model_label']
"""
if ncolumns is None:
ncolumns = 4
def convertfunc(x):
return x
convertfunc = lambda x : x
if ncolumns == 4:
names = ('claimed_id', 'real_id', 'test_label', 'score')
......@@ -410,3 +374,13 @@ def dump_score(filename, score_lines):
else:
raise ValueError("Only scores with 4 and 5 columns are supported.")
numpy.savetxt(filename, score_lines, fmt=fmt)
def _convert_cmc_scores(neg_dict, pos_dict):
"""Converts the negative and positive scores read with :py:func:`cmc_four_column` or :py:func:`cmc_four_column` into a format that is handled by the :py:func:`bob.measure.cmc` and similar functions."""
# convert to lists of tuples of ndarrays (or None)
probe_names = sorted(set(neg_dict.keys()).union(set(pos_dict.keys())))
# get all scores in the desired format
return [(
numpy.array(neg_dict[probe_name], numpy.float64) if probe_name in neg_dict else None,
numpy.array(pos_dict[probe_name], numpy.float64) if probe_name in pos_dict else None
) for probe_name in probe_names]
......@@ -3,10 +3,34 @@
# Chakka Murali Mohan, Trainee, IDIAP Research Institute, Switzerland.
# Mon 23 May 2011 14:36:14 CEST
def log_values(min_step = -4, counts_per_step = 4):
"""log_values(min_step, counts_per_step) -> log_list
This function computes log-scaled values between :math:`10^{M}` and 1 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a negative integer.
The integral ``counts_per_step`` value defines how many values between two adjacent powers of 10 will be created.
The total number of values will be ``-min_step * counts_per_step + 1``.
**Parameters:**
``min_step`` : int (negative)
The power of 10 that will be the minimum value. E.g., the default ``-4`` will result in the first number to be :math:`10^{-4}` = ``0.00001`` or ``0.01%``
``counts_per_step`` : int (positive)
The number of values that will be put between two adjacent powers of 10.
With the default value ``4`` (and default values of ``min_step``), we will get ``log_list[0] == 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``.
**Returns**
``log_list`` : [float]
A list of logarithmically scaled values between :math:`10^{M}` and 1.
"""
import math
return [math.pow(10., i * 1./counts_per_step) for i in range(min_step*counts_per_step,0)] + [1.]
"""Methods to plot error analysis figures such as ROC, precision-recall curve, EPC and DET"""
def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
"""Plots Receiver Operating Charactaristic (ROC) curve.
"""Plots Receiver Operating Characteristic (ROC) curve.
This method will call ``matplotlib`` to plot the ROC curve for a system which
contains a particular set of negatives (impostors) and positives (clients)
......@@ -51,6 +75,46 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
def roc_for_far(negatives, positives, far_values = log_values(), **kwargs):
"""Plots Receiver Operating Characteristic (ROC) curve for the given list of False Acceptance Rates (FAR).
This method will call ``matplotlib`` to plot the ROC curve for a system which
contains a particular set of negatives (impostors) and positives (clients)
scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. All parameters
passed with exception of the three first parameters of this method will be
directly passed to the plot command.
The plot will represent the False Acceptance Rate (FAR) on the horizontal axis and the Correct Acceptance Rate (CAR) on the vertical axis.
The values for the axis will be computed using :py:func:`bob.measure.roc_for_far`.
.. note::
This function does not initiate and save the figure instance, it only
issues the plotting command. You are the responsible for setting up and
saving the figure as you see fit.
**Parameters:**
``negatives, positives`` : array_like(1D, float)
The list of negative and positive scores forwarded to :py:func:`bob.measure.roc`
``far_values`` : [float]
The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
``kwargs`` : keyword arguments
Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
**Returns:**
The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.semilogx`.
"""
from matplotlib import pyplot
from . import roc_for_far as calc
out = calc(negatives, positives, far_values)
return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
"""Plots Precision-Recall curve.
......@@ -341,3 +405,59 @@ def cmc(cmc_scores, logx = True, **kwargs):
pyplot.plot(range(1, len(out)+1), out * 100, **kwargs)
return len(out)
def detection_identification_curve(cmc_scores, far_values = log_values(), rank = 1, logx = True, **kwargs):
"""Plots the Detection & Identification curve over the FAR for the given FAR values.
This curve is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
It requires to have at least one open set probe item, i.e., with no corresponding gallery, such that the positives for that pair are ``None``.
The detection and identification curve first computes FAR thresholds based on the out-of-set probe scores (negative scores).
For each probe item, the **maximum** negative score is used.
Then, it plots the detection and identification rates for those thresholds, which are based on the in-set probe scores only.
See [LiJain2005]_ for more details.
**Parameters:**
``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
See :py:func:`bob.measure.detection_identification_rate`
``far_values`` : [float]
The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].
``rank`` : int or ``None``
The rank for which the curve should be plotted, 1 by default.
``logx`` : bool
Plot the FAR axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``)
``kwargs`` : keyword arguments
Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.
**Returns:**
The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
.. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
"""
import numpy
from matplotlib import pyplot
from . import far_threshold, detection_identification_rate
# for each probe, for which no positives exists, get the highest negative score; and sort them to compute the FAR thresholds
negatives = sorted(max(neg) for neg,pos in cmc_scores if (pos is None or not numpy.array(pos).size) and neg is not None)
if not negatives:
raise ValueError("There need to be at least one pair with only negative scores")
# compute thresholds based on FAR values
thresholds = [far_threshold(negatives, [], v, True) for v in far_values]
# compute detection and identification rate based on the thresholds for the given rank
rates = [100.*detection_identification_rate(cmc_scores, t, rank) for t in thresholds]
# plot curve
if logx:
return pyplot.semilogx(far_values, rates, **kwargs)
else:
return pyplot.plot(far_values, rates, **kwargs)
......@@ -30,6 +30,7 @@ def parse_command_line(command_line_options):
parser.add_argument('-s', '--score-file', required = True, help = 'The score file in 4 or 5 column format to test.')
parser.add_argument('-o', '--output-pdf-file', default = 'cmc.pdf', help = 'The PDF file to write.')
parser.add_argument('-l', '--log-x-scale', action='store_true', help = 'Plot logarithmic Rank axis.')
parser.add_argument('-r', '--rank', type=int, help = 'Plot Detection & Identification rate curve for the given rank instead of the CMC curve.')
parser.add_argument('-x', '--no-plot', action = 'store_true', help = 'Do not print a PDF file, but only report the results.')
parser.add_argument('-p', '--parser', default = '4column', choices = ('4column', '5column'), help = 'The type of the score file.')
......@@ -57,7 +58,7 @@ def main(command_line_options = None):
data = {'4column' : load.cmc_four_column, '5column' : load.cmc_five_column}[args.parser](args.score_file)
# compute recognition rate
rr = recognition_rate(data)
rr = recognition_rate(data, args.rank)
print("Recognition rate for score file", args.score_file, "is %3.2f%%" % (rr * 100))
if not args.no_plot:
......@@ -71,19 +72,34 @@ def main(command_line_options = None):
# CMC
fig = mpl.figure()
max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
mpl.title("CMC Curve")
if args.log_x_scale:
mpl.xlabel('Rank (log)')
if args.rank is None:
max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
mpl.title("CMC Curve")
if args.log_x_scale:
mpl.xlabel('Rank (log)')
else:
mpl.xlabel('Rank')
mpl.ylabel('Recognition Rate in %')
ticks = [int(t) for t in mpl.xticks()[0]]
mpl.xticks(ticks, ticks)
mpl.xlim([1, max_rank])
else:
mpl.xlabel('Rank')
mpl.ylabel('Recognition Rate in %')
plot.detection_identification_curve(data, rank = args.rank, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale)
mpl.title("Detection & Identification Curve")
if args.log_x_scale:
mpl.xlabel('False Acceptance Rate (log) in %')
else:
mpl.xlabel('False Acceptance Rate in %')
mpl.ylabel('Detection & Identification Rate in %')
ticks = ["%s"%(t*100) for t in mpl.xticks()[0]]
mpl.xticks(mpl.xticks()[0], ticks)
mpl.xlim([1e-4, 1])
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.ylim(ymax=101)
# convert log-scale ticks to normal numbers
ticks = [int(t) for t in mpl.xticks()[0]]
mpl.xticks(ticks, ticks)
mpl.xlim([0.9, max_rank + 0.1])
pp.savefig(fig)
pp.close()
......
......@@ -270,7 +270,7 @@ def test_cmc():
# tests the CMC calculation
# test data; should give match characteristics [1/2,1/4,1/3] and CMC [1/3,2/3,1]
test_data = [((0.3, 1.1, 0.5), (0.7)), ((1.4, -1.3, 0.6), (0.2)), ((0.8, 0., 1.5), (-0.8, 1.8)), ((2., 1.3, 1.6, 0.9), (2.4))]
test_data = [((0.3, 1.1, 0.5), (0.7,)), ((1.4, -1.3, 0.6), (0.2,)), ((0.8, 0., 1.5), (-0.8, 1.8)), ((2., 1.3, 1.6, 0.9), (2.4,))]
# compute recognition rate
rr = recognition_rate(test_data)
nose.tools.eq_(rr, 0.5)
......@@ -319,36 +319,34 @@ def test_calibration():
min_cllr = calibration.min_cllr(negatives, positives)
assert min_cllr <= cllr
assert abs(cllr - 3.6183) < 1e-4
assert abs(min_cllr - 0.3373) < 1e-4
assert abs(cllr - 3.61833) < 1e-5, cllr
assert abs(min_cllr - 0.33736) < 1e-5, min_cllr
def test_open_set_recognition_rate():
far_value = 0.01
def test_open_set_rates():
# No error files
cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set.txt"))
normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set.txt"))
assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 1.0) < 1e-8
assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5)) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores) - 1.0) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores) - 7./9.) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 1.0) < 1e-8
t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t) - 1.0) < 1e-8
# One error
cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-one-error.txt"))
normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set-one-error.txt"))
assert abs(bob.measure.recognition_rate(cmc_scores) - 0.857142857143) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 0.857142857143) < 1e-8
t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t) - 0.857142857143) < 1e-8
assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5)) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores) - 6./9.) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
# Two errors
cmc_scores = bob.measure.load.cmc_four_column(F("scores-cmc-4col-open-set-two-errors.txt"))
normal_scores = bob.measure.load.split_four_column(F("scores-cmc-4col-open-set-two-errors.txt"))
assert abs(bob.measure.recognition_rate(cmc_scores) - 0.857142857143) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 0.857142857143) < 1e-8
t = bob.measure.far_threshold(normal_scores[0], normal_scores[1],far_value)
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=t)) < 1e-8
assert abs(bob.measure.detection_identification_rate(cmc_scores, threshold=0.5) - 6./7.) < 1e-8
assert abs(bob.measure.false_alarm_rate(cmc_scores, threshold=0.5) - 0.5) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores) - 6./9.) < 1e-8
assert abs(bob.measure.recognition_rate(cmc_scores, threshold=0.5) - 6./8.) < 1e-8
......@@ -25,6 +25,8 @@ TEST_SCORES_5COL = F('test-5col.txt')
SCORES_4COL_CMC = F('scores-cmc-4col.txt')
SCORES_5COL_CMC = F('scores-cmc-5col.txt')
SCORES_4COL_CMC_OS = F('scores-cmc-4col-open-set.txt')
def test_compute_perf():
# sanity checks
......@@ -68,7 +70,9 @@ def test_compute_cmc():
# sanity checks
assert os.path.exists(SCORES_4COL_CMC)
assert os.path.exists(SCORES_5COL_CMC)
assert os.path.exists(SCORES_4COL_CMC_OS)
from .script.plot_cmc import main
nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC, '--log-x-scale']), 0)
nose.tools.eq_(main(['--self-test', '--score-file', SCORES_5COL_CMC, '--parser', '5column']), 0)
nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC_OS, '--rank', '1']), 0)
......@@ -19,7 +19,8 @@
Methods in the :py:mod:`bob.measure` module can help you to quickly and easily
evaluate error for multi-class or binary classification problems. If you are
not yet familiarized with aspects of performance evaluation, we recommend the
following papers for an overview of some of the methods implemented.
following papers and book chapters for an overview of some of the implemented
methods.
* Bengio, S., Keller, M., Mariéthoz, J. (2004). `The Expected Performance
Curve`_. International Conference on Machine Learning ICML Workshop on ROC
......@@ -27,6 +28,8 @@ following papers for an overview of some of the methods implemented.
* Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
`The DET curve in assessment of detection task performance`_. Fifth European
Conference on Speech Communication and Technology (pp. 1895-1898).
* Li, S., Jain, A.K. (2005), `Handbook of Face Recognition`, Chapter 14, Springer
Overview
--------
......@@ -105,8 +108,8 @@ defined in the first equation.
loaded your scores in two 1D float64 vectors and are ready to evaluate the
performance of the classifier.
Evaluation
----------
Verification
------------
To count the number of correctly classified positives and negatives you can use
the following techniques:
......@@ -171,6 +174,40 @@ calculation of the threshold:
>>> t = bob.measure.min_weighted_error_rate_threshold(negatives, positives, cost, is_sorted = True)
>>> assert T == t
Identification
--------------
For identification, the Recognition Rate is one of the standard measures.
To compute recognition rates, you can use the :py:func:`bob.measure.recognition_rate` function.
This function expects a relatively complex data structure, which is the same as for the `CMC`_ below.
For each probe item, the scores for negative and positive comparisons are computed, and collected for all probe items:
.. doctest::
>>> rr_scores = []
>>> for probe in range(10):
... pos = numpy.random.normal(1, 1, 1)
... neg = numpy.random.normal(0, 1, 19)
... rr_scores.append((neg, pos))
>>> rr = bob.measure.recognition_rate(rr_scores, rank=1)
For open set identification, according to Li and Jain (2005) there are two different error measures defined.
The first measure is the :py:func:`bob.measure.detection_identification_rate`, which counts the number of correctly classified in-gallery probe items.
The second measure is the :py:func:`bob.measure.false_alarm_rate`, which counts, how often an out-of-gallery probe item was incorrectly accepted.
Both rates can be computed using the same data structure, with one exception.
Both functions require that at least one probe item exists, which has no according gallery item, i.e., where the positives are empty or ``None``:
(continued from above...)
.. doctest::
>>> for probe in range(10):
... pos = None
... neg = numpy.random.normal(-2, 1, 10)
... rr_scores.append((neg, pos))
>>> dir = bob.measure.detection_identification_rate(rr_scores, threshold = 0, rank=1)
>>> far = bob.measure.false_alarm_rate(rr_scores, threshold = 0)
Plotting
--------
......@@ -202,6 +239,7 @@ You should see an image like the following one:
.. plot::
import numpy
numpy.random.seed(42)
import bob.measure
from matplotlib import pyplot
......@@ -215,13 +253,13 @@ You should see an image like the following one:
pyplot.title('ROC')
As can be observed, plotting methods live in the namespace
:py:mod:`bob.measure.plot`. They work like `Matplotlib`_'s `plot()`_ method
:py:mod:`bob.measure.plot`. They work like the :py:func:`matplotlib.pyplot.plot`
itself, except that instead of receiving the x and y point coordinates as
parameters, they receive the two :py:class:`numpy.ndarray` arrays with
negatives and positives, as well as an indication of the number of points the
curve must contain.
As in `Matplotlib`_'s `plot()`_ command, you can pass optional parameters for
As in the :py:func:`matplotlib.pyplot.plot` command, you can pass optional parameters for
the line as shown in the example to setup its color, shape and even the label.
For an overview of the keywords accepted, please refer to the `Matplotlib`_'s
Documentation. Other plot properties such as the plot title, axis labels,
......@@ -250,6 +288,7 @@ This will produce an image like the following one:
.. plot::
import numpy
numpy.random.seed(42)
import bob.measure
from matplotlib import pyplot
......@@ -300,6 +339,7 @@ This will produce an image like the following one:
.. plot::
import numpy
numpy.random.seed(42)
import bob.measure
from matplotlib import pyplot
......@@ -323,26 +363,59 @@ The CMC can be calculated from a relatively complex data structure, which define
.. plot::
import numpy
numpy.random.seed(42)
import bob.measure
from matplotlib import pyplot
scores = []
cmc_scores = []
for probe in range(10):
positives = numpy.random.normal(1, 1, 1)
negatives = numpy.random.normal(0, 1, 19)
scores.append((negatives, positives))
bob.measure.plot.cmc(scores, logx=False)
cmc_scores.append((negatives, positives))
bob.measure.plot.cmc(cmc_scores, logx=False)
pyplot.title('CMC')
pyplot.xlabel('Rank')
pyplot.xticks([1,5,10,20])
pyplot.xlim([1,20])
pyplot.ylim([0,100])
pyplot.ylabel('Probability of Recognition (%)')
Usually, there is only a single positive score per probe, but this is not a fixed restriction.
.. note::
The complex data structure can be read from our default 4 or 5 column score files using the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function.
Detection & Identification Curve
================================
The detection & identification curve is designed to evaluate open set identification tasks.
It can be plotted using the :py:func:`bob.measure.plot.detection_identification_curve` function, but it requires at least one open-set probe, i.e., where no corresponding positive score exists, for which the FAR values are computed.
Here, we plot the detection and identification curve for rank 1, so that the recognition rate for FAR=1 will be identical to the rank one :py:func:`bob.measure.recognition_rate` obtained in the CMC plot above.
.. plot::
import numpy
numpy.random.seed(42)
import bob.measure
from matplotlib import pyplot
cmc_scores = []
for probe in range(10):
positives = numpy.random.normal(1, 1, 1)
negatives = numpy.random.normal(0, 1, 19)
cmc_scores.append((negatives, positives))
for probe in range(10):
negatives = numpy.random.normal(-1, 1, 10)
cmc_scores.append((negatives, None))
bob.measure.plot.detection_identification_curve(cmc_scores, rank=1, logx=True)
pyplot.xlabel('False Alarm Rate')
pyplot.ylabel('Detection & Identification Rate (%)')
pyplot.ylim([0,100])
Fine-tunning
============
......@@ -497,5 +570,4 @@ These information are simply stored in the score file, and no further check is a
.. _`The Expected Performance Curve`: http://publications.idiap.ch/downloads/reports/2005/bengio_2005_icml.pdf
.. _`The DET curve in assessment of detection task performance`: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf
.. _`plot()`: http://matplotlib.sourceforge.net/api/pyplot_api.html#matplotlib.pyplot.plot
.. _openbr: http://openbiometrics.org
......@@ -28,6 +28,8 @@ Single point measurements
bob.measure.f_score
bob.measure.precision_recall
bob.measure.recognition_rate
bob.measure.detection_identification_rate
bob.measure.false_alarm_rate
bob.measure.eer_rocch
Thresholds
......@@ -92,6 +94,7 @@ Plotting
bob.measure.plot.epc
bob.measure.plot.precision_recall_curve
bob.measure.plot.cmc
bob.measure.plot.detection_identification_curve
OpenBR conversions
------------------
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment