Merge branch 'multiple-protocols' into 'finalplot'

Add a command for multi protocol (N-fold cross validation) analysis

See merge request !46

bob/pad/base/script/pad_commands.py

@@ -35,7 +35,7 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys):
 
 
 @common_options.metrics_command(common_options.METRICS_HELP.format(
-    names='FtA, FAR, FRR, APCER, BPCER, ACER',
+    names='FtA, APCER, BPCER, FAR, FRR, ACER',
     criteria=CRITERIA, score_format=SCORE_FORMAT,
     command='bob pad metrics'), criteria=CRITERIA)
 def metrics(ctx, scores, evaluation, **kwargs):
@@ -82,3 +82,16 @@ def hist(ctx, scores, evaluation, **kwargs):
 def evaluate(ctx, scores, evaluation, **kwargs):
   common_options.evaluate_flow(
       ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs)
+
+
+@common_options.multi_metrics_command(
+    common_options.MULTI_METRICS_HELP.format(
+        names='FtA, APCER, BPCER, FAR, FRR, ACER',
+        criteria=CRITERIA, score_format=SCORE_FORMAT,
+        command='bob measure multi-metrics'),
+    criteria=CRITERIA)
+def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
+  ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)
+  process = figure.MultiMetrics(
+      ctx, scores, evaluation, load.split)
+  process.run()

bob/pad/base/script/pad_figure.py

@@ -23,6 +23,22 @@ class Metrics(measure_figure.Metrics):
                 criterion, dev_neg, dev_pos, far)
 
 
+class MultiMetrics(measure_figure.MultiMetrics):
+    '''Compute metrics from score files'''
+
+    def __init__(self, ctx, scores, evaluation, func_load):
+        super(MultiMetrics, self).__init__(
+            ctx, scores, evaluation, func_load,
+            names=('FtA', 'APCER', 'BPCER', 'FAR', 'FRR', 'ACER'))
+
+    def get_thres(self, criterion, dev_neg, dev_pos, far):
+        if self._criterion == 'bpcer20':
+            return calc_threshold('bpcer20', dev_neg, dev_pos)
+        else:
+            return super(MultiMetrics, self).get_thres(
+                criterion, dev_neg, dev_pos, far)
+
+
 class Roc(bio_figure.Roc):
     '''ROC for PAD'''
 

setup.py

@@ -140,6 +140,7 @@ setup(
         # bob pad scripts
         'bob.pad.cli': [
             'metrics          = bob.pad.base.script.pad_commands:metrics',
+            'multi-metrics    = bob.pad.base.script.pad_commands:multi_metrics',
             'hist             = bob.pad.base.script.pad_commands:hist',
             'det              = bob.pad.base.script.pad_commands:det',
             'roc              = bob.pad.base.script.pad_commands:roc',