### Add Area Under ROC Curve (AUC)

```Add the function called bob.measure.roc_auc_score.
Add the AUC computation to ``bob measure metrics`` command.```
parent 41a69e7f
 ... ... @@ -474,6 +474,34 @@ def eer(negatives, positives, is_sorted=False, also_farfrr=False): return (far + frr) / 2.0 def roc_auc_score(negatives, positives, npoints=2000, min_far=-8): """Area Under the ROC Curve. Computes the area under the ROC curve. This is useful when you want to report one number that represents an ROC curve. For more information, see: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve Parameters ---------- negatives : array_like The negative scores. positives : array_like The positive scores. npoints : int, optional Number of points in the ROC curve. Higher numbers leads to more accurate ROC. min_far : float, optional Min FAR and FRR values to consider when calculating ROC. Returns ------- float The ROC AUC """ fpr, fnr = roc(negatives, positives, npoints, min_far=min_far) tpr = 1 - fnr area = -1 * numpy.trapz(tpr, fpr) return area def get_config(): """Returns a string containing the configuration information. """ ... ...
 ... ... @@ -13,7 +13,7 @@ CRITERIA = ('eer', 'min-hter', 'far') @common_options.metrics_command( common_options.METRICS_HELP.format( names='FPR, FNR, precision, recall, F1-score', names='FPR, FNR, precision, recall, F1-score, AUC ROC', criteria=CRITERIA, score_format=SCORE_FORMAT, hter_note=' ', command='bob measure metrics'), ... ...
 ... ... @@ -183,7 +183,7 @@ class Metrics(MeasureBase): def __init__(self, ctx, scores, evaluation, func_load, names=('False Positive Rate', 'False Negative Rate', 'Precision', 'Recall', 'F1-score')): 'Precision', 'Recall', 'F1-score', 'Area Under ROC Curve')): super(Metrics, self).__init__(ctx, scores, evaluation, func_load) self.names = names self._tablefmt = ctx.meta.get('tablefmt') ... ... @@ -209,7 +209,7 @@ class Metrics(MeasureBase): return utils.get_thres(criterion, dev_neg, dev_pos, far) def _numbers(self, neg, pos, threshold, fta): from .. import (farfrr, precision_recall, f_score) from .. import (farfrr, precision_recall, f_score, roc_auc_score) # fpr and fnr fmr, fnmr = farfrr(neg, pos, threshold) hter = (fmr + fnmr) / 2.0 ... ... @@ -226,8 +226,11 @@ class Metrics(MeasureBase): # f_score f1_score = f_score(neg, pos, threshold, 1) # AUC ROC auc = roc_auc_score(neg, pos) return (fta, fmr, fnmr, hter, far, frr, fm, ni, fnm, nc, precision, recall, f1_score) recall, f1_score, auc) def _strings(self, metrics): n_dec = '.%df' % self._decimal ... ... @@ -242,9 +245,10 @@ class Metrics(MeasureBase): prec_str = "%s" % format(metrics[10], n_dec) recall_str = "%s" % format(metrics[11], n_dec) f1_str = "%s" % format(metrics[12], n_dec) auc_str = "%s" % format(metrics[13], n_dec) return (fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str, prec_str, recall_str, f1_str) prec_str, recall_str, f1_str, auc_str) def _get_all_metrics(self, idx, input_scores, input_names): ''' Compute all metrics for dev and eval scores''' ... ... @@ -297,11 +301,14 @@ class Metrics(MeasureBase): LOGGER.warn("NaNs scores (%s) were found in %s amd removed", all_metrics[0][0], dev_file) headers = [' ' or title, 'Development'] rows = [[self.names[0], all_metrics[0][1]], [self.names[1], all_metrics[0][2]], [self.names[2], all_metrics[0][6]], [self.names[3], all_metrics[0][7]], [self.names[4], all_metrics[0][8]]] rows = [ [self.names[0], all_metrics[0][1]], [self.names[1], all_metrics[0][2]], [self.names[2], all_metrics[0][6]], [self.names[3], all_metrics[0][7]], [self.names[4], all_metrics[0][8]], [self.names[5], all_metrics[0][9]], ] if self._eval: eval_file = input_names[1] ... ... @@ -317,6 +324,7 @@ class Metrics(MeasureBase): rows[2].append(all_metrics[1][6]) rows[3].append(all_metrics[1][7]) rows[4].append(all_metrics[1][8]) rows[5].append(all_metrics[1][9]) click.echo(tabulate(rows, headers, self._tablefmt), file=self.log_file) ... ...
 ... ... @@ -503,3 +503,17 @@ def test_mindcf(): assert mindcf< 1.0 + 1e-8 def test_roc_auc_score(): from bob.measure import roc_auc_score positives = bob.io.base.load(F('nonsep-positives.hdf5')) negatives = bob.io.base.load(F('nonsep-negatives.hdf5')) auc = roc_auc_score(negatives, positives) # commented out sklearn computation to avoid adding an extra test dependency # from sklearn.metrics import roc_auc_score as oracle_auc # y_true = numpy.concatenate([numpy.ones_like(positives), numpy.zeros_like(negatives)], axis=0) # y_score = numpy.concatenate([positives, negatives], axis=0) # oracle = oracle_auc(y_true, y_score) oracle = 0.9326 assert numpy.allclose(auc, oracle), f"Expected {oracle} but got {auc} instead."
 ... ... @@ -115,7 +115,7 @@ def get_thres(criter, neg, pos, far=None): elif criter == 'far': if far is None: raise ValueError("FAR value must be provided through " "``--far-value`` option.") "``--far-value`` or ``--fpr-value`` option.") from . import far_threshold return far_threshold(neg, pos, far) else: ... ...
 ... ... @@ -284,6 +284,9 @@ town. To plot an ROC curve, in possession of your **negatives** and >>> pyplot.ylabel('FNR (%)') # doctest: +SKIP >>> pyplot.grid(True) >>> pyplot.show() # doctest: +SKIP >>> # You can also compute the area under the ROC curve: >>> bob.measure.roc_auc_score(negatives, positives) 0.8958 You should see an image like the following one: ... ...
 # ignores stuff that does not exist in Python 2.7 manual py:class list # ignores stuff that does not exist but makes sense py:class array py:class array_like py:class optional py:class callable
 ... ... @@ -49,6 +49,7 @@ Curves .. autosummary:: bob.measure.roc bob.measure.roc_auc_score bob.measure.rocch bob.measure.roc_for_far bob.measure.det ... ...
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!