Commit dc0de546 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Change the way the scores arguments are passed to the compute() function: it...

Change the way the scores arguments are passed to the compute() function: it now does not rely on dev,eval pairs anymore and can take any number of different files (e.g. train)
parent 2b95ebd9
Pipeline #19285 passed with stage
in 25 minutes and 28 seconds
...@@ -37,7 +37,7 @@ def metrics(ctx, scores, evaluation, **kwargs): ...@@ -37,7 +37,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
$ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2 $ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
""" """
process = figure.Metrics(ctx, scores, evaluation, load.split_files) process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -76,7 +76,7 @@ def roc(ctx, scores, evaluation, **kwargs): ...@@ -76,7 +76,7 @@ def roc(ctx, scores, evaluation, **kwargs):
$ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1 $ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
""" """
process = figure.Roc(ctx, scores, evaluation, load.split_files) process = figure.Roc(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -114,11 +114,11 @@ def det(ctx, scores, evaluation, **kwargs): ...@@ -114,11 +114,11 @@ def det(ctx, scores, evaluation, **kwargs):
$ bob measure det -o my_det.pdf dev-scores1 eval-scores1 $ bob measure det -o my_det.pdf dev-scores1 eval-scores1
""" """
process = figure.Det(ctx, scores, evaluation, load.split_files) process = figure.Det(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1) @common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf') @common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.title_option() @common_options.title_option()
@common_options.titles_option() @common_options.titles_option()
...@@ -144,7 +144,7 @@ def epc(ctx, scores, **kwargs): ...@@ -144,7 +144,7 @@ def epc(ctx, scores, **kwargs):
$ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1 $ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
""" """
process = figure.Epc(ctx, scores, True, load.split_files) process = figure.Epc(ctx, scores, True, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -184,7 +184,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -184,7 +184,7 @@ def hist(ctx, scores, evaluation, **kwargs):
$ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1 $ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1
""" """
process = figure.Hist(ctx, scores, evaluation, load.split_files) process = figure.Hist(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
......
...@@ -9,17 +9,15 @@ from bob.extension.scripts.click_helper import (bool_option, list_float_option) ...@@ -9,17 +9,15 @@ from bob.extension.scripts.click_helper import (bool_option, list_float_option)
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
def scores_argument(eval_mandatory=False, min_len=1, **kwargs): def scores_argument(min_arg=1, **kwargs):
"""Get the argument for scores, and add `dev-scores` and `eval-scores` in """Get the argument for scores, and add `dev-scores` and `eval-scores` in
the context when `--evaluation` flag is on (default) the context when `--evaluation` flag is on (default)
Parameters Parameters
---------- ----------
eval_mandatory : min_arg : int
If evaluation files are mandatory the minimum number of file needed to evaluate a system. For example,
min_len : PAD functionalities needs licit abd spoof and therefore min_arg = 2
The min lenght of inputs files that are needed. If eval_mandatory is
True, this quantity is multiplied by 2.
Returns Returns
------- -------
...@@ -28,44 +26,27 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs): ...@@ -28,44 +26,27 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
""" """
def custom_scores_argument(func): def custom_scores_argument(func):
def callback(ctx, param, value): def callback(ctx, param, value):
length = len(value) min_a = min_arg or 1
min_arg = min_len or 1 mutli = 1
ctx.meta['min_arg'] = min_arg error = ''
if length < min_arg: if 'evaluation' in ctx.meta and ctx.meta['evaluation']:
mutli += 1
error += '- %d evaluation file(s) \n' % min_a
if 'train' in ctx.meta and ctx.meta['train']:
mutli += 1
error += '- %d training file(s) \n' % min_a
#add more test here if other inputs are needed
min_a *= mutli
ctx.meta['min_arg'] = min_a
if len(value) < 1 or len(value) % ctx.meta['min_arg'] != 0:
raise click.BadParameter( raise click.BadParameter(
'You must provide at least %d score files' % min_arg, 'The number of provided scores must be > 0 and a multiple of %d '
ctx=ctx 'because the following files are required:\n'
'- %d development file(s)\n' % (min_a, min_arg or 1) +
error, ctx=ctx
) )
else: ctx.meta['scores'] = value
ctx.meta['scores'] = value
step = 1
if eval_mandatory or ctx.meta['evaluation']:
step = 2
if (length % (min_arg * 2)) != 0:
pref = 'T' if eval_mandatory else \
('When `--evaluation` flag is on t')
raise click.BadParameter(
'%sest-score(s) must '
'be provided along with dev-score(s). '
'You must provide at least %d score files.' \
% (pref, min_arg * 2), ctx=ctx
)
for arg in range(min_arg):
ctx.meta['dev_scores_%d' % arg] = [
value[i] for i in range(arg * step, length,
min_arg * step)
]
if step > 1:
ctx.meta['eval_scores_%d' % arg] = [
value[i] for i in range((arg * step + 1),
length, min_arg * step)
]
ctx.meta['n_sys'] = len(ctx.meta['dev_scores_0'])
if 'titles' in ctx.meta and \
len(ctx.meta['titles']) != ctx.meta['n_sys']:
raise click.BadParameter(
'#titles not equal to #sytems', ctx=ctx
)
return value return value
return click.argument( return click.argument(
'scores', type=click.Path(exists=True), 'scores', type=click.Path(exists=True),
......
This diff is collapsed.
...@@ -53,6 +53,37 @@ def get_fta(scores): ...@@ -53,6 +53,37 @@ def get_fta(scores):
fta_total += total fta_total += total
return ((neg, pos), fta_sum / fta_total) return ((neg, pos), fta_sum / fta_total)
def get_fta_list(scores):
""" Get FTAs for a list of scores
Parameters
----------
scores: :any:`list`
list of scores
Returns
-------
neg_list: :any:`list`
list of negatives
pos_list: :any:`list`
list of positives
fta_list: :any:`list`
list of FTAs
"""
neg_list = []
pos_list = []
fta_list = []
for score in scores:
neg = pos = fta = None
if score is not None:
(neg, pos), fta = get_fta(score)
if neg is None:
raise ValueError("While loading dev-score file")
neg_list.append(neg)
pos_list.append(pos)
fta_list.append(fta)
return (neg_list, pos_list, fta_list)
def get_thres(criter, neg, pos, far=None): def get_thres(criter, neg, pos, far=None):
"""Get threshold for the given positive/negatives scores and criterion """Get threshold for the given positive/negatives scores and criterion
......
...@@ -114,6 +114,7 @@ Utilities ...@@ -114,6 +114,7 @@ Utilities
.. autosummary:: .. autosummary::
bob.measure.utils.remove_nan bob.measure.utils.remove_nan
bob.measure.utils.get_fta bob.measure.utils.get_fta
bob.measure.utils.get_fta_list
bob.measure.utils.get_thres bob.measure.utils.get_thres
bob.measure.utils.get_colors bob.measure.utils.get_colors
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment