diff --git a/bob/measure/script/figure.py b/bob/measure/script/figure.py index f33a405f0387fc98c8ac5c48180d61f93be23d2b..9369ecd6b961db93986cd1bbd5300b24fdd95109 100644 --- a/bob/measure/script/figure.py +++ b/bob/measure/script/figure.py @@ -14,6 +14,8 @@ from tabulate import tabulate from .. import far_threshold, plot, utils, ppndf import logging +from scipy.stats.stats import ks_2samp + LOGGER = logging.getLogger("bob.measure") @@ -224,6 +226,13 @@ class Metrics(MeasureBase): def get_thres(self, criterion, dev_neg, dev_pos, far): return utils.get_thres(criterion, dev_neg, dev_pos, far) + def get_critical_value(self, neg, pos, alpha=0.05): + c_alpha = numpy.sqrt(-numpy.log(alpha / 2) / 2) + return c_alpha * numpy.sqrt((len(neg) + len(pos)) / (len(neg)*len(pos))) + + def get_cohen_d(self, pos, neg): + return utils.cohen_d(pos, neg) + def _numbers(self, neg, pos, threshold, fta): from .. import farfrr, precision_recall, f_score, roc_auc_score @@ -247,6 +256,14 @@ class Metrics(MeasureBase): # AUC ROC auc = roc_auc_score(neg, pos) auc_log = roc_auc_score(neg, pos, log_scale=True) + + # KS Stat + ks_stat, ks_p = ks_2samp(neg, pos) + ks_crit = self.get_critical_value(neg, pos) + + # Cohen's d test + cohen_d = self.get_cohen_d(neg, pos) + return ( fta, fmr, @@ -263,6 +280,10 @@ class Metrics(MeasureBase): f1_score, auc, auc_log, + ks_stat, + ks_p, + ks_crit, + cohen_d, ) def _strings(self, metrics): @@ -287,6 +308,12 @@ class Metrics(MeasureBase): auc_str = "%s" % format(metrics[13], n_dec) auc_log_str = "%s" % format(metrics[14], n_dec) + ks_stat = f"{metrics[15]:{n_dec}}" + ks_p = f"{metrics[16]:{n_dec}}" + ks_crit = f"{metrics[17]:{n_dec}}" + + cohen_d = f"{metrics[18]:{n_dec}}" + return ( fta_str, fmr_str, @@ -299,6 +326,10 @@ class Metrics(MeasureBase): f1_str, auc_str, auc_log_str, + ks_stat, + ks_p, + ks_crit, + cohen_d, ) def _get_all_metrics(self, idx, input_scores, input_names): diff --git a/bob/measure/utils.py b/bob/measure/utils.py index 8cfcca35a66bd978c4216859b05061c7531a6dd8..248a11cc53cbee4d5893d88a0e9765a5331b47c1 100644 --- a/bob/measure/utils.py +++ b/bob/measure/utils.py @@ -122,6 +122,33 @@ def get_thres(criter, neg, pos, far=None): raise ValueError("Incorrect plotting criterion: ``%s``" % criter) +def cohen_d(x, y): + """ + Get Cohen's d test result computing the effect size between two distributions. + + Parameters + ---------- + x : :py:class:`numpy.ndarray`: + array of scores + y : :py:class:`numpy.ndarray`:: + array of scores + + Returns + ------- + :py:obj:`float` + distance + """ + size_x = len(x) + size_y = len(y) + size = size_x + size_y - 2 + distance = (numpy.mean(x) - numpy.mean(y)) / \ + numpy.sqrt( + ((size_x - 1) * numpy.std(x, ddof=1) ** 2 + (size_y - 1) * numpy.std(y, ddof=1) ** 2) + / size + ) + return distance + + def get_colors(n): """get_colors Get a list of matplotlib colors