Change the way the scores arguments are passed to the compute() function: it...

Change the way the scores arguments are passed to the compute() function: it now does not rely on dev,eval pairs anymore and can take any number of different files (e.g. train)

bob/measure/script/commands.py

@@ -37,7 +37,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
 
         $ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
     """
-    process = figure.Metrics(ctx, scores, evaluation, load.split_files)
+    process = figure.Metrics(ctx, scores, evaluation, load.split)
     process.run()
 
 @click.command()
@@ -76,7 +76,7 @@ def roc(ctx, scores, evaluation, **kwargs):
 
         $ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
     """
-    process = figure.Roc(ctx, scores, evaluation, load.split_files)
+    process = figure.Roc(ctx, scores, evaluation, load.split)
     process.run()
 
 @click.command()
@@ -114,11 +114,11 @@ def det(ctx, scores, evaluation, **kwargs):
 
         $ bob measure det -o my_det.pdf dev-scores1 eval-scores1
     """
-    process = figure.Det(ctx, scores, evaluation, load.split_files)
+    process = figure.Det(ctx, scores, evaluation, load.split)
     process.run()
 
 @click.command()
-@common_options.scores_argument(eval_mandatory=True, nargs=-1)
+@common_options.scores_argument(min_arg=2, nargs=-1)
 @common_options.output_plot_file_option(default_out='epc.pdf')
 @common_options.title_option()
 @common_options.titles_option()
@@ -144,7 +144,7 @@ def epc(ctx, scores, **kwargs):
 
         $ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
     """
-    process = figure.Epc(ctx, scores, True, load.split_files)
+    process = figure.Epc(ctx, scores, True, load.split)
     process.run()
 
 @click.command()
@@ -184,7 +184,7 @@ def hist(ctx, scores, evaluation, **kwargs):
 
         $ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1
     """
-    process = figure.Hist(ctx, scores, evaluation, load.split_files)
+    process = figure.Hist(ctx, scores, evaluation, load.split)
     process.run()
 
 @click.command()

bob/measure/script/common_options.py

@@ -9,17 +9,15 @@ from bob.extension.scripts.click_helper import (bool_option, list_float_option)
 
 LOGGER = logging.getLogger(__name__)
 
-def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
+def scores_argument(min_arg=1, **kwargs):
     """Get the argument for scores, and add `dev-scores` and `eval-scores` in
     the context when `--evaluation` flag is on (default)
 
     Parameters
     ----------
-    eval_mandatory :
-        If evaluation files are mandatory
-    min_len :
-        The min lenght of inputs files that are needed. If eval_mandatory is
-        True, this quantity is multiplied by 2.
+    min_arg : int
+        the minimum number of file needed to evaluate a system. For example,
+        PAD functionalities needs licit abd spoof and therefore min_arg = 2
 
     Returns
     -------
@@ -28,44 +26,27 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
     """
     def custom_scores_argument(func):
         def callback(ctx, param, value):
-            length = len(value)
-            min_arg = min_len or 1
-            ctx.meta['min_arg'] = min_arg
-            if length < min_arg:
+            min_a = min_arg or 1
+            mutli = 1
+            error = ''
+            if 'evaluation' in ctx.meta and ctx.meta['evaluation']:
+                mutli += 1
+                error += '- %d evaluation file(s) \n' % min_a
+            if 'train' in ctx.meta and ctx.meta['train']:
+                mutli += 1
+                error += '- %d training file(s) \n' % min_a
+            #add more test here if other inputs are needed
+
+            min_a *= mutli
+            ctx.meta['min_arg'] = min_a
+            if len(value) < 1 or len(value) % ctx.meta['min_arg'] != 0:
                 raise click.BadParameter(
-                    'You must provide at least %d score files' % min_arg,
-                    ctx=ctx
+                    'The number of provided scores must be > 0 and a multiple of %d '
+                    'because the following files are required:\n'
+                    '- %d development file(s)\n' % (min_a, min_arg or 1) +
+                    error, ctx=ctx
                 )
-            else:
             ctx.meta['scores'] = value
-                step = 1
-                if eval_mandatory or ctx.meta['evaluation']:
-                    step = 2
-                    if (length % (min_arg * 2)) != 0:
-                        pref = 'T' if eval_mandatory else \
-                                ('When `--evaluation` flag is on t')
-                        raise click.BadParameter(
-                            '%sest-score(s) must '
-                            'be provided along with dev-score(s). '
-                            'You must provide at least %d score files.' \
-                            % (pref, min_arg * 2), ctx=ctx
-                        )
-                for arg in range(min_arg):
-                    ctx.meta['dev_scores_%d' % arg] = [
-                        value[i] for i in range(arg * step, length,
-                                                min_arg * step)
-                    ]
-                    if step > 1:
-                        ctx.meta['eval_scores_%d' % arg] = [
-                            value[i] for i in range((arg * step + 1),
-                                                    length, min_arg * step)
-                        ]
-                ctx.meta['n_sys'] = len(ctx.meta['dev_scores_0'])
-                if 'titles' in ctx.meta and \
-                   len(ctx.meta['titles']) != ctx.meta['n_sys']:
-                    raise click.BadParameter(
-                        '#titles not equal to #sytems', ctx=ctx
-                    )
             return value
         return click.argument(
             'scores', type=click.Path(exists=True),

bob/measure/utils.py

@@ -53,6 +53,37 @@ def get_fta(scores):
     fta_total += total
     return ((neg, pos), fta_sum / fta_total)
 
+def get_fta_list(scores):
+    """ Get FTAs for a list of scores
+
+    Parameters
+    ----------
+    scores: :any:`list`
+        list of scores
+
+    Returns
+    -------
+    neg_list: :any:`list`
+        list of negatives
+    pos_list: :any:`list`
+        list of positives
+    fta_list: :any:`list`
+        list of FTAs
+    """
+    neg_list = []
+    pos_list = []
+    fta_list = []
+    for score in scores:
+        neg = pos = fta = None
+        if score is not None:
+            (neg, pos), fta = get_fta(score)
+            if neg is None:
+                raise ValueError("While loading dev-score file")
+        neg_list.append(neg)
+        pos_list.append(pos)
+        fta_list.append(fta)
+    return (neg_list, pos_list, fta_list)
+
 def get_thres(criter, neg, pos, far=None):
     """Get threshold for the given positive/negatives scores and criterion
 

doc/py_api.rst

@@ -114,6 +114,7 @@ Utilities
 .. autosummary::
    bob.measure.utils.remove_nan
    bob.measure.utils.get_fta
+   bob.measure.utils.get_fta_list
    bob.measure.utils.get_thres
    bob.measure.utils.get_colors