Commit 73a7b02e authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Add commands to compute bio specific metrics

parent 3dde93a6
Pipeline #18455 failed with stage
in 48 minutes and 22 seconds
......@@ -8,22 +8,22 @@ from ..score import load
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
FUNC_CMC = lambda x: load.load_files(x, load.cmc)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
@common_options.cost_option()
@common_options.thresholds_option()
@common_options.far_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, test, **kargs):
def metrics(ctx, scores, criter, test, **kargs):
"""Prints a single output line that contains all info for a given
criterion (eer or hter).
......@@ -44,7 +44,10 @@ def metrics(ctx, scores, test, **kargs):
$ bob bio metrics --test {dev,test}-scores1 {dev,test}-scores2
"""
process = measure_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
if criter == 'rr':
process = bio_figure.Metrics(ctx, scores, test, FUNC_CMC)
else:
process = bio_figure.Metrics(ctx, scores, test, FUNC_SPLIT)
process.run()
@click.command()
......@@ -154,7 +157,7 @@ def epc(ctx, scores, **kargs):
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.threshold_option()
@common_options.thresholds_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kwargs):
......
'''Plots and measures for bob.bio.base'''
import click
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
import bob.measure
from bob.measure import plot
from tabulate import tabulate
class Cmc(measure_figure.PlotBase):
''' Handles the plotting of Cmc
......@@ -108,3 +111,96 @@ class Dic(measure_figure.PlotBase):
color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14],
label=self._label('development', dev_file, idx)
)
class Metrics(measure_figure.Metrics):
''' Compute metrics from score files'''
def init_process(self):
if self._criter == 'rr':
self._thres = [None] * len(self.dev_names) if self._thres is None else \
self._thres
def compute(self, idx, dev_score, dev_file=None,
test_score=None, test_file=None):
''' Compute metrics for the given criteria'''
headers = ['', 'Development %s' % dev_file]
if self._test and test_score is not None:
headers.append('Test % s' % test_file)
if self._criter == 'rr':
rr = bob.measure.recognition_rate(dev_score, self._thres[idx])
dev_rr = "%.3f%%" % (100 * rr)
raws = [['RR', dev_rr]]
if self._test and test_score is not None:
rr = bob.measure.recognition_rate(test_score, self._thres[idx])
test_rr = "%.3f%%" % (100 * rr)
raws[0].append(test_rr)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
elif self._criter == 'mindcf':
if 'cost' in self._ctx.meta:
cost = 0.99 if 'cost' not in self._ctx.meta else\
self._ctx.meta['cost']
threshold = bob.measure.min_weighted_error_rate_threshold(
dev_score[0], dev_score[1], cost
) if self._thres is None else self._thres[idx]
if self._thres is None:
click.echo(
"[minDCF - Cost:%f] Threshold on Development set `%s`: %e"\
% (cost, dev_file, threshold),
file=self.log_file
)
else:
click.echo(
"[minDCF] User defined Threshold: %e" % threshold,
file=self.log_file
)
# apply threshold to development set
far, frr = bob.measure.farfrr(
dev_score[0], dev_score[1], threshold
)
dev_far_str = "%.3f%%" % (100 * far)
dev_frr_str = "%.3f%%" % (100 * frr)
dev_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
raws = [['FAR', dev_far_str],
['FRR', dev_frr_str],
['minDCF', dev_mindcf_str]]
if self._test and test_score is not None:
# apply threshold to development set
far, frr = bob.measure.farfrr(
test_score[0], test_score[1], threshold
)
test_far_str = "%.3f%%" % (100 * far)
test_frr_str = "%.3f%%" % (100 * frr)
test_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
raws[0].append(test_far_str)
raws[1].append(test_frr_str)
raws[2].append(test_mindcf_str)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
elif self._criter == 'cllr':
cllr = bob.measure.calibration.cllr(dev_score[0], dev_score[1])
min_cllr = bob.measure.calibration.min_cllr(
dev_score[0], dev_score[1]
)
dev_cllr_str = "%.3f%%" % cllr
dev_min_cllr_str = "%.3f%%" % min_cllr
raws = [['Cllr', dev_cllr_str],
['minCllr', dev_min_cllr_str]]
if self._test and test_score is not None:
cllr = bob.measure.calibration.cllr(test_score[0],
test_score[1])
min_cllr = bob.measure.calibration.min_cllr(
test_score[0], test_score[1]
)
test_cllr_str = "%.3f%%" % cllr
test_min_cllr_str = "%.3f%%" % min_cllr
raws[0].append(test_cllr_str)
raws[1].append(test_min_cllr_str)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
else:
super(Metrics, self).compute(
idx, dev_score, dev_file, test_score, test_file
)
......@@ -37,10 +37,53 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--test', dev1, dev2]
commands.metrics, ['-l', 'tmp', '--test', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '-t', '-T', '0.1',
'--criter', 'mindcf', '--cost', 0.9,
dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp',
'--criter', 'mindcf', '--cost', 0.9,
dev1]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-t', '--criter', 'cllr', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criter', 'cllr', '--cost', 0.9,
dev1]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-t', '--criter', 'rr', '-T',
'0.1', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criter', 'rr', dev1, dev2]
)
assert result.exit_code == 0
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment