Commit dc0de546 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Change the way the scores arguments are passed to the compute() function: it...

Change the way the scores arguments are passed to the compute() function: it now does not rely on dev,eval pairs anymore and can take any number of different files (e.g. train)
parent 2b95ebd9
Pipeline #19285 passed with stage
in 25 minutes and 28 seconds
......@@ -37,7 +37,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
$ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
"""
process = figure.Metrics(ctx, scores, evaluation, load.split_files)
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
@click.command()
......@@ -76,7 +76,7 @@ def roc(ctx, scores, evaluation, **kwargs):
$ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
"""
process = figure.Roc(ctx, scores, evaluation, load.split_files)
process = figure.Roc(ctx, scores, evaluation, load.split)
process.run()
@click.command()
......@@ -114,11 +114,11 @@ def det(ctx, scores, evaluation, **kwargs):
$ bob measure det -o my_det.pdf dev-scores1 eval-scores1
"""
process = figure.Det(ctx, scores, evaluation, load.split_files)
process = figure.Det(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1)
@common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.title_option()
@common_options.titles_option()
......@@ -144,7 +144,7 @@ def epc(ctx, scores, **kwargs):
$ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
"""
process = figure.Epc(ctx, scores, True, load.split_files)
process = figure.Epc(ctx, scores, True, load.split)
process.run()
@click.command()
......@@ -184,7 +184,7 @@ def hist(ctx, scores, evaluation, **kwargs):
$ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1
"""
process = figure.Hist(ctx, scores, evaluation, load.split_files)
process = figure.Hist(ctx, scores, evaluation, load.split)
process.run()
@click.command()
......
......@@ -9,17 +9,15 @@ from bob.extension.scripts.click_helper import (bool_option, list_float_option)
LOGGER = logging.getLogger(__name__)
def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
def scores_argument(min_arg=1, **kwargs):
"""Get the argument for scores, and add `dev-scores` and `eval-scores` in
the context when `--evaluation` flag is on (default)
Parameters
----------
eval_mandatory :
If evaluation files are mandatory
min_len :
The min lenght of inputs files that are needed. If eval_mandatory is
True, this quantity is multiplied by 2.
min_arg : int
the minimum number of file needed to evaluate a system. For example,
PAD functionalities needs licit abd spoof and therefore min_arg = 2
Returns
-------
......@@ -28,44 +26,27 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
"""
def custom_scores_argument(func):
def callback(ctx, param, value):
length = len(value)
min_arg = min_len or 1
ctx.meta['min_arg'] = min_arg
if length < min_arg:
min_a = min_arg or 1
mutli = 1
error = ''
if 'evaluation' in ctx.meta and ctx.meta['evaluation']:
mutli += 1
error += '- %d evaluation file(s) \n' % min_a
if 'train' in ctx.meta and ctx.meta['train']:
mutli += 1
error += '- %d training file(s) \n' % min_a
#add more test here if other inputs are needed
min_a *= mutli
ctx.meta['min_arg'] = min_a
if len(value) < 1 or len(value) % ctx.meta['min_arg'] != 0:
raise click.BadParameter(
'You must provide at least %d score files' % min_arg,
ctx=ctx
'The number of provided scores must be > 0 and a multiple of %d '
'because the following files are required:\n'
'- %d development file(s)\n' % (min_a, min_arg or 1) +
error, ctx=ctx
)
else:
ctx.meta['scores'] = value
step = 1
if eval_mandatory or ctx.meta['evaluation']:
step = 2
if (length % (min_arg * 2)) != 0:
pref = 'T' if eval_mandatory else \
('When `--evaluation` flag is on t')
raise click.BadParameter(
'%sest-score(s) must '
'be provided along with dev-score(s). '
'You must provide at least %d score files.' \
% (pref, min_arg * 2), ctx=ctx
)
for arg in range(min_arg):
ctx.meta['dev_scores_%d' % arg] = [
value[i] for i in range(arg * step, length,
min_arg * step)
]
if step > 1:
ctx.meta['eval_scores_%d' % arg] = [
value[i] for i in range((arg * step + 1),
length, min_arg * step)
]
ctx.meta['n_sys'] = len(ctx.meta['dev_scores_0'])
if 'titles' in ctx.meta and \
len(ctx.meta['titles']) != ctx.meta['n_sys']:
raise click.BadParameter(
'#titles not equal to #sytems', ctx=ctx
)
ctx.meta['scores'] = value
return value
return click.argument(
'scores', type=click.Path(exists=True),
......
This diff is collapsed.
......@@ -53,6 +53,37 @@ def get_fta(scores):
fta_total += total
return ((neg, pos), fta_sum / fta_total)
def get_fta_list(scores):
""" Get FTAs for a list of scores
Parameters
----------
scores: :any:`list`
list of scores
Returns
-------
neg_list: :any:`list`
list of negatives
pos_list: :any:`list`
list of positives
fta_list: :any:`list`
list of FTAs
"""
neg_list = []
pos_list = []
fta_list = []
for score in scores:
neg = pos = fta = None
if score is not None:
(neg, pos), fta = get_fta(score)
if neg is None:
raise ValueError("While loading dev-score file")
neg_list.append(neg)
pos_list.append(pos)
fta_list.append(fta)
return (neg_list, pos_list, fta_list)
def get_thres(criter, neg, pos, far=None):
"""Get threshold for the given positive/negatives scores and criterion
......
......@@ -114,6 +114,7 @@ Utilities
.. autosummary::
bob.measure.utils.remove_nan
bob.measure.utils.get_fta
bob.measure.utils.get_fta_list
bob.measure.utils.get_thres
bob.measure.utils.get_colors
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment