Skip to content
Snippets Groups Projects
Commit 6ea7c229 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Made changes according to the modifications of the bob.measure base class... 

Made changes according to the modifications of the bob.measure base class (more generic implementation)
parent ca805a3a
No related branches found
No related tags found
2 merge requests!146Add 4-5-col files related functionalities and add click commands,!143Set of click commands for bio base
Pipeline #
Hide whitespace changes
Inline Side-by-side
bob/bio/base/script/commands.py
+ 9
− 11
View file @ 6ea7c229
...@@ -8,8 +8,6 @@ from bob.measure.script import common_options ...@@ -8,8 +8,6 @@ from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option, from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option) open_file_mode_option)
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
FUNC_CMC = lambda x: load.load_files(x, load.cmc)
def rank_option(**kwargs): def rank_option(**kwargs):
'''Get option for rank parameter''' '''Get option for rank parameter'''
...@@ -62,9 +60,9 @@ def metrics(ctx, scores, evaluation, **kargs): ...@@ -62,9 +60,9 @@ def metrics(ctx, scores, evaluation, **kargs):
$ bob bio metrics {dev,eval}-scores1 {dev,eval}-scores2 $ bob bio metrics {dev,eval}-scores1 {dev,eval}-scores2
""" """
if 'criter' in ctx.meta and ctx.meta['criter'] == 'rr': if 'criter' in ctx.meta and ctx.meta['criter'] == 'rr':
process = bio_figure.Metrics(ctx, scores, evaluation, FUNC_CMC) process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc)
else: else:
process = bio_figure.Metrics(ctx, scores, evaluation, FUNC_SPLIT) process = bio_figure.Metrics(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -106,7 +104,7 @@ def roc(ctx, scores, evaluation, **kargs): ...@@ -106,7 +104,7 @@ def roc(ctx, scores, evaluation, **kargs):
$ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1 $ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1
""" """
process = bio_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT) process = bio_figure.Roc(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -146,11 +144,11 @@ def det(ctx, scores, evaluation, **kargs): ...@@ -146,11 +144,11 @@ def det(ctx, scores, evaluation, **kargs):
$ bob bio det -o my_det.pdf dev-scores1 eval-scores1 $ bob bio det -o my_det.pdf dev-scores1 eval-scores1
""" """
process = bio_figure.Det(ctx, scores, evaluation, FUNC_SPLIT) process = bio_figure.Det(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1) @common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf') @common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option() @common_options.titles_option()
@common_options.points_curve_option() @common_options.points_curve_option()
...@@ -175,7 +173,7 @@ def epc(ctx, scores, **kargs): ...@@ -175,7 +173,7 @@ def epc(ctx, scores, **kargs):
$ bob bio epc -o my_epc.pdf dev-scores1 eval-scores1 $ bob bio epc -o my_epc.pdf dev-scores1 eval-scores1
""" """
process = measure_figure.Epc(ctx, scores, True, FUNC_SPLIT) process = measure_figure.Epc(ctx, scores, True, load.split)
process.run() process.run()
@click.command() @click.command()
...@@ -215,7 +213,7 @@ def cmc(ctx, scores, evaluation, **kargs): ...@@ -215,7 +213,7 @@ def cmc(ctx, scores, evaluation, **kargs):
$ bob bio cmc -o my_roc.pdf dev-scores1 eval-scores1 $ bob bio cmc -o my_roc.pdf dev-scores1 eval-scores1
""" """
process = bio_figure.Cmc(ctx, scores, evaluation, FUNC_CMC) process = bio_figure.Cmc(ctx, scores, evaluation, load.cmc)
process.run() process.run()
@click.command() @click.command()
...@@ -264,7 +262,7 @@ def dic(ctx, scores, evaluation, **kargs): ...@@ -264,7 +262,7 @@ def dic(ctx, scores, evaluation, **kargs):
$ bob bio dic -o my_roc.pdf dev-scores1 eval-scores1 $ bob bio dic -o my_roc.pdf dev-scores1 eval-scores1
""" """
process = bio_figure.Dic(ctx, scores, evaluation, FUNC_CMC) process = bio_figure.Dic(ctx, scores, evaluation, load.cmc)
process.run() process.run()
@click.command() @click.command()
...@@ -306,7 +304,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -306,7 +304,7 @@ def hist(ctx, scores, evaluation, **kwargs):
$ bob bio hist --criter --show-dev hter dev-scores1 eval-scores1 $ bob bio hist --criter --show-dev hter dev-scores1 eval-scores1
""" """
process = bio_figure.Hist(ctx, scores, evaluation, FUNC_SPLIT) process = bio_figure.Hist(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command() @click.command()
......
bob/bio/base/script/figure.py
+ 36
− 40
View file @ 6ea7c229
...@@ -34,17 +34,16 @@ class Cmc(measure_figure.PlotBase): ...@@ -34,17 +34,16 @@ class Cmc(measure_figure.PlotBase):
self._y_label = self._y_label or 'Identification rate' self._y_label = self._y_label or 'Identification rate'
self._max_R = 0 self._max_R = 0
def compute(self, idx, dev_score, dev_file=None, def compute(self, idx, input_scores, input_names):
eval_score=None, eval_file=None):
''' Plot CMC for dev and eval data using ''' Plot CMC for dev and eval data using
:py:func:`bob.measure.plot.cmc`''' :py:func:`bob.measure.plot.cmc`'''
mpl.figure(1) mpl.figure(1)
if self._eval: if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14] linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc( rank = plot.cmc(
dev_score, logx=self._semilogx, input_scores[0], logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('development', dev_file, idx) label=self._label('development', input_names[0], idx)
) )
self._max_R = max(rank, self._max_R) self._max_R = max(rank, self._max_R)
linestyle = '--' linestyle = '--'
...@@ -53,16 +52,16 @@ class Cmc(measure_figure.PlotBase): ...@@ -53,16 +52,16 @@ class Cmc(measure_figure.PlotBase):
linestyle = measure_figure.LINESTYLES[idx % 14] linestyle = measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc( rank = plot.cmc(
eval_score, logx=self._semilogx, input_scores[1], logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', eval_file, idx) label=self._label('eval', input_names[1], idx)
) )
self._max_R = max(rank, self._max_R) self._max_R = max(rank, self._max_R)
else: else:
rank = plot.cmc( rank = plot.cmc(
dev_score, logx=self._semilogx, input_scores[0], logx=self._semilogx,
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14], color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', dev_file, idx) label=self._label('development', input_names[0], idx)
) )
self._max_R = max(rank, self._max_R) self._max_R = max(rank, self._max_R)
...@@ -77,17 +76,16 @@ class Dic(measure_figure.PlotBase): ...@@ -77,17 +76,16 @@ class Dic(measure_figure.PlotBase):
self._x_label = self._title or 'FAR' self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR' self._y_label = self._title or 'DIR'
def compute(self, idx, dev_score, dev_file=None, def compute(self, idx, input_scores, input_names):
eval_score=None, eval_file=None):
''' Plot DIC for dev and eval data using ''' Plot DIC for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`''' :py:func:`bob.measure.plot.detection_identification_curve`'''
mpl.figure(1) mpl.figure(1)
if self._eval: if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14] linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve( plot.detection_identification_curve(
dev_score, rank=self._rank, logx=self._semilogx, input_scores[0], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('development', dev_file, idx) label=self._label('development', input_names[0], idx)
) )
linestyle = '--' linestyle = '--'
if self._split: if self._split:
...@@ -95,37 +93,36 @@ class Dic(measure_figure.PlotBase): ...@@ -95,37 +93,36 @@ class Dic(measure_figure.PlotBase):
linestyle = measure_figure.LINESTYLES[idx % 14] linestyle = measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve( plot.detection_identification_curve(
eval_score, rank=self._rank, logx=self._semilogx, input_scores[1], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle, color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', eval_file, idx) label=self._label('eval', input_names[1], idx)
) )
else: else:
rank = plot.detection_identification_curve( plot.detection_identification_curve(
dev_score, rank=self._rank, logx=self._semilogx, input_scores[0], rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14], color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', dev_file, idx) label=self._label('development', input_names[0], idx)
) )
class Metrics(measure_figure.Metrics): class Metrics(measure_figure.Metrics):
''' Compute metrics from score files''' ''' Compute metrics from score files'''
def init_process(self): def init_process(self):
if self._criter == 'rr': if self._criter == 'rr':
self._thres = [None] * self.n_sytem if self._thres is None else \ self._thres = [None] * self.n_systems if self._thres is None else \
self._thres self._thres
def compute(self, idx, dev_score, dev_file=None, def compute(self, idx, input_scores, input_names):
eval_score=None, eval_file=None):
''' Compute metrics for the given criteria''' ''' Compute metrics for the given criteria'''
title = self._titles[idx] if self._titles is not None else None title = self._titles[idx] if self._titles is not None else None
headers = ['' or title, 'Development %s' % dev_file] headers = ['' or title, 'Development %s' % input_names[0]]
if self._eval and eval_score is not None: if self._eval and input_scores[1] is not None:
headers.append('eval % s' % eval_file) headers.append('eval % s' % input_names[1])
if self._criter == 'rr': if self._criter == 'rr':
rr = bob.measure.recognition_rate(dev_score, self._thres[idx]) rr = bob.measure.recognition_rate(input_scores[0], self._thres[idx])
dev_rr = "%.1f%%" % (100 * rr) dev_rr = "%.1f%%" % (100 * rr)
raws = [['RR', dev_rr]] raws = [['RR', dev_rr]]
if self._eval and eval_score is not None: if self._eval and input_scores[1] is not None:
rr = bob.measure.recognition_rate(eval_score, self._thres[idx]) rr = bob.measure.recognition_rate(input_scores[1], self._thres[idx])
eval_rr = "%.1f%%" % (100 * rr) eval_rr = "%.1f%%" % (100 * rr)
raws[0].append(eval_rr) raws[0].append(eval_rr)
click.echo( click.echo(
...@@ -136,12 +133,12 @@ class Metrics(measure_figure.Metrics): ...@@ -136,12 +133,12 @@ class Metrics(measure_figure.Metrics):
cost = 0.99 if 'cost' not in self._ctx.meta else\ cost = 0.99 if 'cost' not in self._ctx.meta else\
self._ctx.meta['cost'] self._ctx.meta['cost']
threshold = bob.measure.min_weighted_error_rate_threshold( threshold = bob.measure.min_weighted_error_rate_threshold(
dev_score[0], dev_score[1], cost input_scores[0][0], input_scores[0][1], cost
) if self._thres is None else self._thres[idx] ) if self._thres is None else self._thres[idx]
if self._thres is None: if self._thres is None:
click.echo( click.echo(
"[minDCF - Cost:%f] Threshold on Development set `%s`: %e"\ "[minDCF - Cost:%f] Threshold on Development set `%s`: %e"\
% (cost, dev_file, threshold), % (cost, input_names[0], threshold),
file=self.log_file file=self.log_file
) )
else: else:
...@@ -151,7 +148,7 @@ class Metrics(measure_figure.Metrics): ...@@ -151,7 +148,7 @@ class Metrics(measure_figure.Metrics):
) )
# apply threshold to development set # apply threshold to development set
far, frr = bob.measure.farfrr( far, frr = bob.measure.farfrr(
dev_score[0], dev_score[1], threshold input_scores[0][0], input_scores[0][1], threshold
) )
dev_far_str = "%.1f%%" % (100 * far) dev_far_str = "%.1f%%" % (100 * far)
dev_frr_str = "%.1f%%" % (100 * frr) dev_frr_str = "%.1f%%" % (100 * frr)
...@@ -159,10 +156,10 @@ class Metrics(measure_figure.Metrics): ...@@ -159,10 +156,10 @@ class Metrics(measure_figure.Metrics):
raws = [['FAR', dev_far_str], raws = [['FAR', dev_far_str],
['FRR', dev_frr_str], ['FRR', dev_frr_str],
['minDCF', dev_mindcf_str]] ['minDCF', dev_mindcf_str]]
if self._eval and eval_score is not None: if self._eval and input_scores[1] is not None:
# apply threshold to development set # apply threshold to development set
far, frr = bob.measure.farfrr( far, frr = bob.measure.farfrr(
eval_score[0], eval_score[1], threshold input_scores[1][0], input_scores[1][1], threshold
) )
eval_far_str = "%.1f%%" % (100 * far) eval_far_str = "%.1f%%" % (100 * far)
eval_frr_str = "%.1f%%" % (100 * frr) eval_frr_str = "%.1f%%" % (100 * frr)
...@@ -174,19 +171,20 @@ class Metrics(measure_figure.Metrics): ...@@ -174,19 +171,20 @@ class Metrics(measure_figure.Metrics):
tabulate(raws, headers, self._tablefmt), file=self.log_file tabulate(raws, headers, self._tablefmt), file=self.log_file
) )
elif self._criter == 'cllr': elif self._criter == 'cllr':
cllr = bob.measure.calibration.cllr(dev_score[0], dev_score[1]) cllr = bob.measure.calibration.cllr(input_scores[0][0],
input_scores[0][1])
min_cllr = bob.measure.calibration.min_cllr( min_cllr = bob.measure.calibration.min_cllr(
dev_score[0], dev_score[1] input_scores[0][0], input_scores[0][1]
) )
dev_cllr_str = "%.1f%%" % cllr dev_cllr_str = "%.1f%%" % cllr
dev_min_cllr_str = "%.1f%%" % min_cllr dev_min_cllr_str = "%.1f%%" % min_cllr
raws = [['Cllr', dev_cllr_str], raws = [['Cllr', dev_cllr_str],
['minCllr', dev_min_cllr_str]] ['minCllr', dev_min_cllr_str]]
if self._eval and eval_score is not None: if self._eval and input_scores[1] is not None:
cllr = bob.measure.calibration.cllr(eval_score[0], cllr = bob.measure.calibration.cllr(input_scores[1][0],
eval_score[1]) input_scores[1][1])
min_cllr = bob.measure.calibration.min_cllr( min_cllr = bob.measure.calibration.min_cllr(
eval_score[0], eval_score[1] input_scores[1][0], input_scores[1][1]
) )
eval_cllr_str = "%.1f%%" % cllr eval_cllr_str = "%.1f%%" % cllr
eval_min_cllr_str = "%.1f%%" % min_cllr eval_min_cllr_str = "%.1f%%" % min_cllr
...@@ -196,9 +194,7 @@ class Metrics(measure_figure.Metrics): ...@@ -196,9 +194,7 @@ class Metrics(measure_figure.Metrics):
tabulate(raws, headers, self._tablefmt), file=self.log_file tabulate(raws, headers, self._tablefmt), file=self.log_file
) )
else: else:
super(Metrics, self).compute( super(Metrics, self).compute(idx, input_scores, input_names)
idx, dev_score, dev_file, eval_score, eval_file
)
class Hist(measure_figure.Hist): class Hist(measure_figure.Hist):
''' Histograms for biometric scores ''' ''' Histograms for biometric scores '''
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment