Merge branch 'correct-apcer-calculation' into 'master'

Correct apcer calculation

Closes #31

See merge request !60

bob/pad/base/algorithm/MLP.py

@@ -7,11 +7,8 @@ from bob.pad.base.algorithm import Algorithm
 import bob.learn.mlp
 import bob.io.base
 
-from bob.bio.video.utils import FrameContainer
-from bob.pad.base.utils import convert_frame_cont_to_array
-
-from bob.core.log import setup
-logger = setup("bob.pad.base")
+import logging
+logger = logging.getLogger(__name__)
 
 
 class MLP(Algorithm):
@@ -42,11 +39,11 @@ class MLP(Algorithm):
           criterion to stop the training: if the difference
           between current and last loss is smaller than
           this number, then stop training.
-        
+
         """
-        Algorithm.__init__(self,  
+        Algorithm.__init__(self,
                            performs_projection=True,
-                           requires_projector_training=True, 
+                           requires_projector_training=True,
                            **kwargs)
 
         self.hidden_units = hidden_units
@@ -54,32 +51,31 @@ class MLP(Algorithm):
         self.precision = precision
         self.mlp = None
 
-
     def train_projector(self, training_features, projector_file):
         """Trains the MLP
 
         Parameters
         ----------
-        training_features : :any:`list` of :py:class:`numpy.ndarray` 
+        training_features : :any:`list` of :py:class:`numpy.ndarray`
           Data used to train the MLP. The real attempts are in training_features[0] and the attacks are in training_features[1]
         projector_file : str
           Filename where to save the trained model.
-        
+
         """
         # training is done in batch (i.e. using all training data)
         batch_size = len(training_features[0]) + len(training_features[1])
-        
+
         # The labels
         label_real = numpy.zeros((len(training_features[0]), 2), dtype='float64')
         label_real[:, 0] = 1
         label_attack = numpy.zeros((len(training_features[1]), 2), dtype='float64')
         label_attack[:, 1] = 0
-        
+
         real = numpy.array(training_features[0])
         attack = numpy.array(training_features[1])
         X = numpy.vstack([real, attack])
         Y = numpy.vstack([label_real, label_attack])
-      
+
         # Building MLP architecture
         input_dim = real.shape[1]
         shape = []
@@ -89,16 +85,16 @@ class MLP(Algorithm):
         # last layer contains two units: one for each class (i.e. real and attack)
         shape.append(2)
         shape = tuple(shape)
-    
+
         self.mlp = bob.learn.mlp.Machine(shape)
         self.mlp.output_activation = bob.learn.activation.Logistic()
         self.mlp.randomize()
         trainer = bob.learn.mlp.BackProp(batch_size, bob.learn.mlp.CrossEntropyLoss(self.mlp.output_activation), self.mlp, train_biases=True)
-    
+
         n_iter = 0
         previous_cost = 0
         current_cost = 1
-        while (n_iter < self.max_iter) and (abs(previous_cost - current_cost) > self.precision): 
+        while (n_iter < self.max_iter) and (abs(previous_cost - current_cost) > self.precision):
             previous_cost = current_cost
             trainer.train(self.mlp, X, Y)
             current_cost = trainer.cost(self.mlp, X, Y)
@@ -107,14 +103,13 @@ class MLP(Algorithm):
 
         f = bob.io.base.HDF5File(projector_file, 'w')
         self.mlp.save(f)
-    
 
     def project(self, feature):
         """Project the given feature
-        
+
         Parameters
         ----------
-        feature : :py:class:`numpy.ndarray` 
+        feature : :py:class:`numpy.ndarray`
           The feature to classify
 
         Returns
@@ -126,18 +121,17 @@ class MLP(Algorithm):
        #     feature = convert_frame_cont_to_array(feature)
         return self.mlp(feature)
 
-
     def score(self, toscore):
         """Returns the probability of the real class.
 
         Parameters
         ----------
         toscore : :py:class:`numpy.ndarray`
-       
+
         Returns
         -------
         float
-         probability of the authentication attempt to be real. 
+         probability of the authentication attempt to be real.
         """
         if toscore.ndim == 1:
             return [toscore[0]]

bob/pad/base/script/pad_commands.py

@@ -7,93 +7,235 @@ import bob.bio.base.script.gen as bio_gen
 import bob.measure.script.figure as measure_figure
 from bob.bio.base.score import load
 from . import pad_figure as figure
+from .error_utils import negatives_per_pai_and_positives
+from functools import partial
 
 SCORE_FORMAT = (
-    "Files must be 4-col format, see "
-    ":py:func:`bob.bio.base.score.load.four_column`.")
-CRITERIA = ('eer', 'min-hter', 'bpcer20')
+    "Files must be 4-col format, see " ":py:func:`bob.bio.base.score.load.four_column`."
+)
+CRITERIA = (
+    "eer",
+    "min-hter",
+    "far",
+    "bpcer5000",
+    "bpcer2000",
+    "bpcer1000",
+    "bpcer500",
+    "bpcer200",
+    "bpcer100",
+    "bpcer50",
+    "bpcer20",
+    "bpcer10",
+    "bpcer5",
+    "bpcer2",
+    "bpcer1",
+)
+
+
+def metrics_option(
+    sname="-m",
+    lname="--metrics",
+    name="metrics",
+    help="List of metrics to print. Provide a string with comma separated metric "
+    "names. For possible values see the default value.",
+    default="apcer_pais,apcer,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score",
+    **kwargs
+):
+    """The metrics option"""
+
+    def custom_metrics_option(func):
+        def callback(ctx, param, value):
+            if value is not None:
+                value = value.split(",")
+            ctx.meta[name] = value
+            return value
+
+        return click.option(
+            sname,
+            lname,
+            default=default,
+            help=help,
+            show_default=True,
+            callback=callback,
+            **kwargs
+        )(func)
+
+    return custom_metrics_option
+
+
+def regexps_option(
+    help="A list of regular expressions (by repeating this option) to be used to "
+    "categorize PAIs. Each regexp must match one type of PAI.",
+    **kwargs
+):
+    def custom_regexps_option(func):
+        def callback(ctx, param, value):
+            ctx.meta["regexps"] = value
+            return value
+
+        return click.option(
+            "-r",
+            "--regexps",
+            default=None,
+            multiple=True,
+            help=help,
+            callback=callback,
+            **kwargs
+        )(func)
+
+    return custom_regexps_option
+
+
+def regexp_column_option(
+    help="The column in the score files to match the regular expressions against.",
+    **kwargs
+):
+    def custom_regexp_column_option(func):
+        def callback(ctx, param, value):
+            ctx.meta["regexp_column"] = value
+            return value
+
+        return click.option(
+            "-rc",
+            "--regexp-column",
+            default="real_id",
+            type=click.Choice(("claimed_id", "real_id", "test_label")),
+            help=help,
+            show_default=True,
+            callback=callback,
+            **kwargs
+        )(func)
+
+    return custom_regexp_column_option
 
 
 @click.command()
-@click.argument('outdir')
-@click.option('-mm', '--mean-match', default=10, type=click.FLOAT,
-              show_default=True)
-@click.option('-mnm', '--mean-non-match', default=-10,
-              type=click.FLOAT, show_default=True)
-@click.option('-n', '--n-sys', default=1, type=click.INT, show_default=True)
+@click.argument("outdir")
+@click.option("-mm", "--mean-match", default=10, type=click.FLOAT, show_default=True)
+@click.option(
+    "-mnm", "--mean-non-match", default=-10, type=click.FLOAT, show_default=True
+)
+@click.option("-n", "--n-sys", default=1, type=click.INT, show_default=True)
 @verbosity_option()
 @click.pass_context
 def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
-  """Generate random scores.
-  Generates random scores in 4col or 5col format. The scores are generated
-  using Gaussian distribution whose mean is an input
-  parameter. The generated scores can be used as hypothetical datasets.
-  Invokes :py:func:`bob.bio.base.script.commands.gen`.
-  """
-  ctx.meta['five_col'] = False
-  ctx.forward(bio_gen.gen)
-
-
-@common_options.metrics_command(common_options.METRICS_HELP.format(
-    names='FtA, APCER, BPCER, FAR, FRR, ACER',
-    criteria=CRITERIA, score_format=SCORE_FORMAT,
-    hter_note='Note that FAR = APCER * (1 - FtA), '
-    'FRR = FtA + BPCER * (1 - FtA) and ACER = (APCER + BPCER) / 2.',
-    command='bob pad metrics'), criteria=CRITERIA)
-def metrics(ctx, scores, evaluation, **kwargs):
-  process = figure.Metrics(ctx, scores, evaluation, load.split)
-  process.run()
+    """Generate random scores.
+    Generates random scores in 4col or 5col format. The scores are generated
+    using Gaussian distribution whose mean is an input
+    parameter. The generated scores can be used as hypothetical datasets.
+    Invokes :py:func:`bob.bio.base.script.commands.gen`.
+    """
+    ctx.meta["five_col"] = False
+    ctx.forward(bio_gen.gen)
+
+
+@common_options.metrics_command(
+    common_options.METRICS_HELP.format(
+        names="FtA, APCER, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
+        criteria=CRITERIA,
+        score_format=SCORE_FORMAT,
+        hter_note="Note that APCER = max(APCER_pais), BPCER=FNR, "
+        "FAR = FPR * (1 - FtA), "
+        "FRR = FtA + FNR * (1 - FtA), "
+        "ACER = (APCER + BPCER) / 2, "
+        "and HTER = (FPR + FNR) / 2. "
+        "You can control which metrics are printed using the --metrics option. "
+        "You can use --regexps and --regexp_column options to change the behavior "
+        "of finding Presentation Attack Instrument (PAI) types",
+        command="bob pad metrics",
+    ),
+    criteria=CRITERIA,
+    epilog="""\b
+More Examples:
+\b
+bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer,bpcer,acer,hter \
+/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}
+
+See also ``bob pad multi-metrics``.
+""",
+)
+@regexps_option()
+@regexp_column_option()
+@metrics_option()
+def metrics(ctx, scores, evaluation, regexps, regexp_column, metrics, **kwargs):
+    load_fn = partial(
+        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
+    )
+    process = figure.Metrics(ctx, scores, evaluation, load_fn, metrics)
+    process.run()
 
 
 @common_options.roc_command(
-    common_options.ROC_HELP.format(
-        score_format=SCORE_FORMAT, command='bob pad roc'))
+    common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob pad roc")
+)
 def roc(ctx, scores, evaluation, **kwargs):
-  process = figure.Roc(ctx, scores, evaluation, load.split)
-  process.run()
+    process = figure.Roc(ctx, scores, evaluation, load.split)
+    process.run()
 
 
 @common_options.det_command(
-    common_options.DET_HELP.format(
-        score_format=SCORE_FORMAT, command='bob pad det'))
+    common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob pad det")
+)
 def det(ctx, scores, evaluation, **kwargs):
-  process = figure.Det(ctx, scores, evaluation, load.split)
-  process.run()
+    process = figure.Det(ctx, scores, evaluation, load.split)
+    process.run()
 
 
 @common_options.epc_command(
-    common_options.EPC_HELP.format(
-        score_format=SCORE_FORMAT, command='bob pad epc'))
+    common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob pad epc")
+)
 def epc(ctx, scores, **kwargs):
-  process = measure_figure.Epc(ctx, scores, True, load.split, hter='ACER')
-  process.run()
+    process = measure_figure.Epc(ctx, scores, True, load.split, hter="ACER")
+    process.run()
 
 
 @common_options.hist_command(
-    common_options.HIST_HELP.format(
-        score_format=SCORE_FORMAT, command='bob pad hist'))
+    common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob pad hist")
+)
 def hist(ctx, scores, evaluation, **kwargs):
-  process = figure.Hist(ctx, scores, evaluation, load.split)
-  process.run()
+    process = figure.Hist(ctx, scores, evaluation, load.split)
+    process.run()
 
 
 @common_options.evaluate_command(
     common_options.EVALUATE_HELP.format(
-        score_format=SCORE_FORMAT, command='bob pad evaluate'),
-    criteria=CRITERIA)
+        score_format=SCORE_FORMAT, command="bob pad evaluate"
+    ),
+    criteria=CRITERIA,
+)
 def evaluate(ctx, scores, evaluation, **kwargs):
-  common_options.evaluate_flow(
-      ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs)
+    common_options.evaluate_flow(
+        ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
+    )
 
 
 @common_options.multi_metrics_command(
     common_options.MULTI_METRICS_HELP.format(
-        names='FtA, APCER, BPCER, FAR, FRR, ACER',
-        criteria=CRITERIA, score_format=SCORE_FORMAT,
-        command='bob pad multi-metrics'),
-    criteria=CRITERIA)
-def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
-  ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)
-  process = figure.MultiMetrics(
-      ctx, scores, evaluation, load.split)
-  process.run()
+        names="FtA, APCER, BPCER, FAR, FRR, ACER, HTER, precision, recall, f1_score",
+        criteria=CRITERIA,
+        score_format=SCORE_FORMAT,
+        command="bob pad multi-metrics",
+    ),
+    criteria=CRITERIA,
+    epilog="""\b
+More examples:
+
+\b
+bob pad multi-metrics -vvv -e -pn 6 -lg IQM,LBP -r print -r video \
+/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_3_{1,2,3,4,5,6}/scores/scores-{dev,eval}
+
+See also ``bob pad metrics``.
+""",
+)
+@regexps_option()
+@regexp_column_option()
+@metrics_option(default="fta,apcer_pais,apcer,bpcer,acer,hter")
+def multi_metrics(
+    ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs
+):
+    ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
+    load_fn = partial(
+        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
+    )
+    process = figure.MultiMetrics(ctx, scores, evaluation, load_fn, metrics)
+    process.run()

bob/pad/base/script/pad_figure.py

-
-'''Runs error analysis on score sets, outputs metrics and plots'''
+"""Runs error analysis on score sets, outputs metrics and plots"""
 
 import bob.measure.script.figure as measure_figure
+from bob.measure.utils import get_fta_list
+from bob.measure import farfrr, precision_recall, f_score
 import bob.bio.base.script.figure as bio_figure
-from .error_utils import calc_threshold
+from .error_utils import calc_threshold, apcer_bpcer
+import click
+from tabulate import tabulate
+import numpy as np
+
 
-ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')
+def _normalize_input_scores(input_score, input_name):
+    pos, negs = input_score
+    # convert scores to sorted numpy arrays and keep a copy of all negatives
+    pos = np.ascontiguousarray(pos)
+    pos.sort()
+    all_negs = np.ascontiguousarray([s for neg in negs.values() for s in neg])
+    all_negs.sort()
+    # FTA is calculated on pos and all_negs so we remove nans from negs
+    for k, v in negs.items():
+        v = np.ascontiguousarray(v)
+        v.sort()
+        negs[k] = v[~np.isnan(v)]
+    neg_list, pos_list, fta_list = get_fta_list([(all_negs, pos)])
+    all_negs, pos, fta = neg_list[0], pos_list[0], fta_list[0]
+    return input_name, pos, negs, all_negs, fta
 
 
 class Metrics(bio_figure.Metrics):
-    '''Compute metrics from score files'''
+    """Compute metrics from score files"""
+
+    def __init__(self, ctx, scores, evaluation, func_load, names):
+        if isinstance(names, str):
+            names = names.split(",")
+        super(Metrics, self).__init__(ctx, scores, evaluation, func_load, names)
+
+    def get_thres(self, criterion, pos, negs, all_negs, far_value):
+        return calc_threshold(
+            criterion,
+            pos=pos,
+            negs=negs.values(),
+            all_negs=all_negs,
+            far_value=far_value,
+            is_sorted=True,
+        )
+
+    def _numbers(self, threshold, pos, negs, all_negs, fta):
+        pais = list(negs.keys())
+        apcer_pais, apcer, bpcer = apcer_bpcer(threshold, pos, *[negs[k] for k in pais])
+        apcer_pais = {k: apcer_pais[i] for i, k in enumerate(pais)}
+        acer = (apcer + bpcer) / 2.0
+        fpr, fnr = farfrr(all_negs, pos, threshold)
+        hter = (fpr + fnr) / 2.0
+        far = fpr * (1 - fta)
+        frr = fta + fnr * (1 - fta)
+
+        nn = all_negs.shape[0]  # number of attack
+        fp = int(round(fpr * nn))  # number of false positives
+        np = pos.shape[0]  # number of bonafide
+        fn = int(round(fnr * np))  # number of false negatives
+
+        # precision and recall
+        precision, recall = precision_recall(all_negs, pos, threshold)
+
+        # f_score
+        f1_score = f_score(all_negs, pos, threshold, 1)
+        metrics = dict(
+            apcer_pais=apcer_pais,
+            apcer=apcer,
+            bpcer=bpcer,
+            acer=acer,
+            fta=fta,
+            fpr=fpr,
+            fnr=fnr,
+            hter=hter,
+            far=far,
+            frr=frr,
+            fp=fp,
+            nn=nn,
+            fn=fn,
+            np=np,
+            precision=precision,
+            recall=recall,
+            f1_score=f1_score,
+        )
+        return metrics
 
-    def __init__(self, ctx, scores, evaluation, func_load,
-                names=('FtA', 'APCER', 'BPCER', 'FAR', 'FRR', 'HTER')):
-        super(Metrics, self).__init__(
-            ctx, scores, evaluation, func_load, names
+    def _strings(self, metrics):
+        n_dec = ".%df" % self._decimal
+        for k, v in metrics.items():
+            if k in ("precision", "recall", "f1_score"):
+                metrics[k] = "%s" % format(v, n_dec)
+            elif k in ("np", "nn", "fp", "fn"):
+                continue
+            elif k in ("fpr", "fnr"):
+                if "fp" in metrics:
+                    metrics[k] = "%s%% (%d/%d)" % (
+                        format(100 * v, n_dec),
+                        metrics["fp" if k == "fpr" else "fn"],
+                        metrics["np" if k == "fpr" else "nn"],
+                    )
+                else:
+                    metrics[k] = "%s%%" % format(100 * v, n_dec)
+            elif k == "apcer_pais":
+                metrics[k] = {
+                    k1: "%s%%" % format(100 * v1, n_dec) for k1, v1 in v.items()
+                }
+            else:
+                metrics[k] = "%s%%" % format(100 * v, n_dec)
+
+        return metrics
+
+    def _get_all_metrics(self, idx, input_scores, input_names):
+        """ Compute all metrics for dev and eval scores"""
+        for i, (score, name) in enumerate(zip(input_scores, input_names)):
+            input_scores[i] = _normalize_input_scores(score, name)
+
+        dev_file, dev_pos, dev_negs, dev_all_negs, dev_fta = input_scores[0]
+        if self._eval:
+            eval_file, eval_pos, eval_negs, eval_all_negs, eval_fta = input_scores[1]
+
+        threshold = (
+            self.get_thres(self._criterion, dev_pos, dev_negs, dev_all_negs, self._far)
+            if self._thres is None
+            else self._thres[idx]
+        )
+
+        title = self._legends[idx] if self._legends is not None else None
+        if self._thres is None:
+            far_str = ""
+            if self._criterion == "far" and self._far is not None:
+                far_str = str(self._far)
+            click.echo(
+                "[Min. criterion: %s %s] Threshold on Development set `%s`: %e"
+                % (self._criterion.upper(), far_str, title or dev_file, threshold),
+                file=self.log_file,
+            )
+        else:
+            click.echo(
+                "[Min. criterion: user provided] Threshold on "
+                "Development set `%s`: %e" % (dev_file or title, threshold),
+                file=self.log_file,
+            )
+
+        res = []
+        res.append(
+            self._strings(
+                self._numbers(threshold, dev_pos, dev_negs, dev_all_negs, dev_fta)
+            )
         )
 
-    def get_thres(self, criterion, dev_neg, dev_pos, far):
-        if self._criterion == 'bpcer20':
-            return calc_threshold('bpcer20', dev_neg, dev_pos)
+        if self._eval:
+            # computes statistics for the eval set based on the threshold a priori
+            res.append(
+                self._strings(
+                    self._numbers(
+                        threshold, eval_pos, eval_negs, eval_all_negs, eval_fta
+                    )
+                )
+            )
         else:
-            return super(Metrics, self).get_thres(
-                criterion, dev_neg, dev_pos, far)
+            res.append(None)
 
+        return res