Commit eae2a343 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'roc-auc' into 'master'

Add Area Under ROC Curve (AUC)

Closes #2

See merge request !97
parents 41a69e7f a6400ba9
Pipeline #35291 canceled with stages
in 14 minutes and 47 seconds
......@@ -474,6 +474,44 @@ def eer(negatives, positives, is_sorted=False, also_farfrr=False):
return (far + frr) / 2.0
def roc_auc_score(negatives, positives, npoints=2000, min_far=-8, log_scale=False):
"""Area Under the ROC Curve.
Computes the area under the ROC curve. This is useful when you want to report one
number that represents an ROC curve. This implementation uses the trapezoidal rule for the integration of the ROC curve. For more information, see:
https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve
Parameters
----------
negatives : array_like
The negative scores.
positives : array_like
The positive scores.
npoints : int, optional
Number of points in the ROC curve. Higher numbers leads to more accurate ROC.
min_far : float, optional
Min FAR and FRR values to consider when calculating ROC.
log_scale : bool, optional
If True, converts the x axis (FPR) to log10 scale before calculating AUC. This is
useful in cases where len(negatives) >> len(positives)
Returns
-------
float
The ROC AUC. If ``log_scale`` is False, the value should be between 0 and 1.
"""
fpr, fnr = roc(negatives, positives, npoints, min_far=min_far)
tpr = 1 - fnr
if log_scale:
fpr_pos = fpr > 0
fpr, tpr = fpr[fpr_pos], tpr[fpr_pos]
fpr = numpy.log10(fpr)
area = -1 * numpy.trapz(tpr, fpr)
return area
def get_config():
"""Returns a string containing the configuration information.
"""
......
......@@ -13,7 +13,7 @@ CRITERIA = ('eer', 'min-hter', 'far')
@common_options.metrics_command(
common_options.METRICS_HELP.format(
names='FPR, FNR, precision, recall, F1-score',
names='FPR, FNR, precision, recall, F1-score, AUC ROC',
criteria=CRITERIA, score_format=SCORE_FORMAT,
hter_note=' ',
command='bob measure metrics'),
......
......@@ -183,7 +183,7 @@ class Metrics(MeasureBase):
def __init__(self, ctx, scores, evaluation, func_load,
names=('False Positive Rate', 'False Negative Rate',
'Precision', 'Recall', 'F1-score')):
'Precision', 'Recall', 'F1-score', 'Area Under ROC Curve', 'Area Under ROC Curve (log scale)')):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
self.names = names
self._tablefmt = ctx.meta.get('tablefmt')
......@@ -209,7 +209,7 @@ class Metrics(MeasureBase):
return utils.get_thres(criterion, dev_neg, dev_pos, far)
def _numbers(self, neg, pos, threshold, fta):
from .. import (farfrr, precision_recall, f_score)
from .. import (farfrr, precision_recall, f_score, roc_auc_score)
# fpr and fnr
fmr, fnmr = farfrr(neg, pos, threshold)
hter = (fmr + fnmr) / 2.0
......@@ -226,8 +226,12 @@ class Metrics(MeasureBase):
# f_score
f1_score = f_score(neg, pos, threshold, 1)
# AUC ROC
auc = roc_auc_score(neg, pos)
auc_log = roc_auc_score(neg, pos, log_scale=True)
return (fta, fmr, fnmr, hter, far, frr, fm, ni, fnm, nc, precision,
recall, f1_score)
recall, f1_score, auc, auc_log)
def _strings(self, metrics):
n_dec = '.%df' % self._decimal
......@@ -242,9 +246,11 @@ class Metrics(MeasureBase):
prec_str = "%s" % format(metrics[10], n_dec)
recall_str = "%s" % format(metrics[11], n_dec)
f1_str = "%s" % format(metrics[12], n_dec)
auc_str = "%s" % format(metrics[13], n_dec)
auc_log_str = "%s" % format(metrics[14], n_dec)
return (fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str,
prec_str, recall_str, f1_str)
prec_str, recall_str, f1_str, auc_str, auc_log_str)
def _get_all_metrics(self, idx, input_scores, input_names):
''' Compute all metrics for dev and eval scores'''
......@@ -297,11 +303,15 @@ class Metrics(MeasureBase):
LOGGER.warn("NaNs scores (%s) were found in %s amd removed",
all_metrics[0][0], dev_file)
headers = [' ' or title, 'Development']
rows = [[self.names[0], all_metrics[0][1]],
[self.names[1], all_metrics[0][2]],
[self.names[2], all_metrics[0][6]],
[self.names[3], all_metrics[0][7]],
[self.names[4], all_metrics[0][8]]]
rows = [
[self.names[0], all_metrics[0][1]],
[self.names[1], all_metrics[0][2]],
[self.names[2], all_metrics[0][6]],
[self.names[3], all_metrics[0][7]],
[self.names[4], all_metrics[0][8]],
[self.names[5], all_metrics[0][9]],
[self.names[6], all_metrics[0][10]],
]
if self._eval:
eval_file = input_names[1]
......@@ -317,6 +327,8 @@ class Metrics(MeasureBase):
rows[2].append(all_metrics[1][6])
rows[3].append(all_metrics[1][7])
rows[4].append(all_metrics[1][8])
rows[5].append(all_metrics[1][9])
rows[6].append(all_metrics[1][10])
click.echo(tabulate(rows, headers, self._tablefmt), file=self.log_file)
......
......@@ -503,3 +503,22 @@ def test_mindcf():
assert mindcf< 1.0 + 1e-8
def test_roc_auc_score():
from bob.measure import roc_auc_score
positives = bob.io.base.load(F('nonsep-positives.hdf5'))
negatives = bob.io.base.load(F('nonsep-negatives.hdf5'))
auc = roc_auc_score(negatives, positives)
# commented out sklearn computation to avoid adding an extra test dependency
# from sklearn.metrics import roc_auc_score as oracle_auc
# y_true = numpy.concatenate([numpy.ones_like(positives), numpy.zeros_like(negatives)], axis=0)
# y_score = numpy.concatenate([positives, negatives], axis=0)
# oracle = oracle_auc(y_true, y_score)
oracle = 0.9326
assert numpy.allclose(auc, oracle), f"Expected {oracle} but got {auc} instead."
# test the function on log scale as well
auc = roc_auc_score(negatives, positives, log_scale=True)
oracle = 1.4183699583300993
assert numpy.allclose(auc, oracle), f"Expected {oracle} but got {auc} instead."
......@@ -115,7 +115,7 @@ def get_thres(criter, neg, pos, far=None):
elif criter == 'far':
if far is None:
raise ValueError("FAR value must be provided through "
"``--far-value`` option.")
"``--far-value`` or ``--fpr-value`` option.")
from . import far_threshold
return far_threshold(neg, pos, far)
else:
......
......@@ -284,6 +284,9 @@ town. To plot an ROC curve, in possession of your **negatives** and
>>> pyplot.ylabel('FNR (%)') # doctest: +SKIP
>>> pyplot.grid(True)
>>> pyplot.show() # doctest: +SKIP
>>> # You can also compute the area under the ROC curve:
>>> bob.measure.roc_auc_score(negatives, positives)
0.8958
You should see an image like the following one:
......
# ignores stuff that does not exist in Python 2.7 manual
py:class list
# ignores stuff that does not exist but makes sense
py:class array
py:class array_like
py:class optional
py:class callable
......@@ -49,6 +49,7 @@ Curves
.. autosummary::
bob.measure.roc
bob.measure.roc_auc_score
bob.measure.rocch
bob.measure.roc_for_far
bob.measure.det
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment