Commit d61e7d62 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Add Area Under ROC Curve (AUC)

Add the function called bob.measure.roc_auc_score.
Add the AUC computation to ``bob measure metrics`` command.
parent 41a69e7f
......@@ -474,6 +474,34 @@ def eer(negatives, positives, is_sorted=False, also_farfrr=False):
return (far + frr) / 2.0
def roc_auc_score(negatives, positives, npoints=2000, min_far=-8):
"""Area Under the ROC Curve.
Computes the area under the ROC curve. This is useful when you want to report one
number that represents an ROC curve. For more information, see:
https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve
Parameters
----------
negatives : array_like
The negative scores.
positives : array_like
The positive scores.
npoints : int, optional
Number of points in the ROC curve. Higher numbers leads to more accurate ROC.
min_far : float, optional
Min FAR and FRR values to consider when calculating ROC.
Returns
-------
float
The ROC AUC
"""
fpr, fnr = roc(negatives, positives, npoints, min_far=min_far)
tpr = 1 - fnr
area = -1 * numpy.trapz(tpr, fpr)
return area
def get_config():
"""Returns a string containing the configuration information.
"""
......
......@@ -13,7 +13,7 @@ CRITERIA = ('eer', 'min-hter', 'far')
@common_options.metrics_command(
common_options.METRICS_HELP.format(
names='FPR, FNR, precision, recall, F1-score',
names='FPR, FNR, precision, recall, F1-score, AUC ROC',
criteria=CRITERIA, score_format=SCORE_FORMAT,
hter_note=' ',
command='bob measure metrics'),
......
......@@ -183,7 +183,7 @@ class Metrics(MeasureBase):
def __init__(self, ctx, scores, evaluation, func_load,
names=('False Positive Rate', 'False Negative Rate',
'Precision', 'Recall', 'F1-score')):
'Precision', 'Recall', 'F1-score', 'Area Under ROC Curve')):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
self.names = names
self._tablefmt = ctx.meta.get('tablefmt')
......@@ -209,7 +209,7 @@ class Metrics(MeasureBase):
return utils.get_thres(criterion, dev_neg, dev_pos, far)
def _numbers(self, neg, pos, threshold, fta):
from .. import (farfrr, precision_recall, f_score)
from .. import (farfrr, precision_recall, f_score, roc_auc_score)
# fpr and fnr
fmr, fnmr = farfrr(neg, pos, threshold)
hter = (fmr + fnmr) / 2.0
......@@ -226,8 +226,11 @@ class Metrics(MeasureBase):
# f_score
f1_score = f_score(neg, pos, threshold, 1)
# AUC ROC
auc = roc_auc_score(neg, pos)
return (fta, fmr, fnmr, hter, far, frr, fm, ni, fnm, nc, precision,
recall, f1_score)
recall, f1_score, auc)
def _strings(self, metrics):
n_dec = '.%df' % self._decimal
......@@ -242,9 +245,10 @@ class Metrics(MeasureBase):
prec_str = "%s" % format(metrics[10], n_dec)
recall_str = "%s" % format(metrics[11], n_dec)
f1_str = "%s" % format(metrics[12], n_dec)
auc_str = "%s" % format(metrics[13], n_dec)
return (fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str,
prec_str, recall_str, f1_str)
prec_str, recall_str, f1_str, auc_str)
def _get_all_metrics(self, idx, input_scores, input_names):
''' Compute all metrics for dev and eval scores'''
......@@ -297,11 +301,14 @@ class Metrics(MeasureBase):
LOGGER.warn("NaNs scores (%s) were found in %s amd removed",
all_metrics[0][0], dev_file)
headers = [' ' or title, 'Development']
rows = [[self.names[0], all_metrics[0][1]],
[self.names[1], all_metrics[0][2]],
[self.names[2], all_metrics[0][6]],
[self.names[3], all_metrics[0][7]],
[self.names[4], all_metrics[0][8]]]
rows = [
[self.names[0], all_metrics[0][1]],
[self.names[1], all_metrics[0][2]],
[self.names[2], all_metrics[0][6]],
[self.names[3], all_metrics[0][7]],
[self.names[4], all_metrics[0][8]],
[self.names[5], all_metrics[0][9]],
]
if self._eval:
eval_file = input_names[1]
......@@ -317,6 +324,7 @@ class Metrics(MeasureBase):
rows[2].append(all_metrics[1][6])
rows[3].append(all_metrics[1][7])
rows[4].append(all_metrics[1][8])
rows[5].append(all_metrics[1][9])
click.echo(tabulate(rows, headers, self._tablefmt), file=self.log_file)
......
......@@ -503,3 +503,17 @@ def test_mindcf():
assert mindcf< 1.0 + 1e-8
def test_roc_auc_score():
from bob.measure import roc_auc_score
positives = bob.io.base.load(F('nonsep-positives.hdf5'))
negatives = bob.io.base.load(F('nonsep-negatives.hdf5'))
auc = roc_auc_score(negatives, positives)
# commented out sklearn computation to avoid adding an extra test dependency
# from sklearn.metrics import roc_auc_score as oracle_auc
# y_true = numpy.concatenate([numpy.ones_like(positives), numpy.zeros_like(negatives)], axis=0)
# y_score = numpy.concatenate([positives, negatives], axis=0)
# oracle = oracle_auc(y_true, y_score)
oracle = 0.9326
assert numpy.allclose(auc, oracle), f"Expected {oracle} but got {auc} instead."
......@@ -115,7 +115,7 @@ def get_thres(criter, neg, pos, far=None):
elif criter == 'far':
if far is None:
raise ValueError("FAR value must be provided through "
"``--far-value`` option.")
"``--far-value`` or ``--fpr-value`` option.")
from . import far_threshold
return far_threshold(neg, pos, far)
else:
......
......@@ -284,6 +284,9 @@ town. To plot an ROC curve, in possession of your **negatives** and
>>> pyplot.ylabel('FNR (%)') # doctest: +SKIP
>>> pyplot.grid(True)
>>> pyplot.show() # doctest: +SKIP
>>> # You can also compute the area under the ROC curve:
>>> bob.measure.roc_auc_score(negatives, positives)
0.8958
You should see an image like the following one:
......
# ignores stuff that does not exist in Python 2.7 manual
py:class list
# ignores stuff that does not exist but makes sense
py:class array
py:class array_like
py:class optional
py:class callable
......@@ -49,6 +49,7 @@ Curves
.. autosummary::
bob.measure.roc
bob.measure.roc_auc_score
bob.measure.rocch
bob.measure.roc_for_far
bob.measure.det
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment