Commit 4b7707ea authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Change test->eval, change tests accordingly, add titles option to metrics abd hist

parent dcc72296
Pipeline #18761 failed with stage
in 26 minutes and 38 seconds
This diff is collapsed.
......@@ -16,8 +16,8 @@ class Cmc(measure_figure.PlotBase):
_semilogx: :obj:`bool`
If true (default), X-axis will be semilog10
'''
def __init__(self, ctx, scores, test, func_load):
super(Cmc, self).__init__(ctx, scores, test, func_load)
def __init__(self, ctx, scores, evaluation, func_load):
super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._title = 'CMC'
......@@ -26,11 +26,11 @@ class Cmc(measure_figure.PlotBase):
self._max_R = 0
def compute(self, idx, dev_score, dev_file=None,
test_score=None, test_file=None):
eval_score=None, eval_file=None):
''' Plot CMC for dev and eval data using
:py:func:`bob.measure.plot.cmc`'''
mpl.figure(1)
if self._test:
if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc(
dev_score, logx=self._semilogx,
......@@ -44,9 +44,9 @@ class Cmc(measure_figure.PlotBase):
linestyle = measure_figure.LINESTYLES[idx % 14]
rank = plot.cmc(
test_score, logx=self._semilogx,
eval_score, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('test', test_file, idx)
label=self._label('eval', eval_file, idx)
)
self._max_R = max(rank, self._max_R)
else:
......@@ -74,8 +74,8 @@ class Dic(measure_figure.PlotBase):
_rank: :obj:`int`
Rank to be used to plot DIC (default: 1)
'''
def __init__(self, ctx, scores, test, func_load):
super(Dic, self).__init__(ctx, scores, test, func_load)
def __init__(self, ctx, scores, evaluation, func_load):
super(Dic, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
......@@ -84,11 +84,11 @@ class Dic(measure_figure.PlotBase):
self._y_label = 'DIR'
def compute(self, idx, dev_score, dev_file=None,
test_score=None, test_file=None):
eval_score=None, eval_file=None):
''' Plot DIC for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`'''
mpl.figure(1)
if self._test:
if self._eval:
linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve(
dev_score, rank=self._rank, logx=self._semilogx,
......@@ -101,9 +101,9 @@ class Dic(measure_figure.PlotBase):
linestyle = measure_figure.LINESTYLES[idx % 14]
plot.detection_identification_curve(
test_score, rank=self._rank, logx=self._semilogx,
eval_score, rank=self._rank, logx=self._semilogx,
color=self._colors[idx], linestyle=linestyle,
label=self._label('test', test_file, idx)
label=self._label('eval', eval_file, idx)
)
else:
rank = plot.detection_identification_curve(
......@@ -116,23 +116,24 @@ class Metrics(measure_figure.Metrics):
''' Compute metrics from score files'''
def init_process(self):
if self._criter == 'rr':
self._thres = [None] * len(self.dev_names) if self._thres is None else \
self._thres = [None] * self.n_sytem if self._thres is None else \
self._thres
def compute(self, idx, dev_score, dev_file=None,
test_score=None, test_file=None):
eval_score=None, eval_file=None):
''' Compute metrics for the given criteria'''
headers = ['', 'Development %s' % dev_file]
if self._test and test_score is not None:
headers.append('Test % s' % test_file)
title = self._titles[idx] if self._titles is not None else None
headers = ['' or title, 'Development %s' % dev_file]
if self._eval and eval_score is not None:
headers.append('eval % s' % eval_file)
if self._criter == 'rr':
rr = bob.measure.recognition_rate(dev_score, self._thres[idx])
dev_rr = "%.3f%%" % (100 * rr)
dev_rr = "%.1f%%" % (100 * rr)
raws = [['RR', dev_rr]]
if self._test and test_score is not None:
rr = bob.measure.recognition_rate(test_score, self._thres[idx])
test_rr = "%.3f%%" % (100 * rr)
raws[0].append(test_rr)
if self._eval and eval_score is not None:
rr = bob.measure.recognition_rate(eval_score, self._thres[idx])
eval_rr = "%.1f%%" % (100 * rr)
raws[0].append(eval_rr)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
......@@ -158,23 +159,23 @@ class Metrics(measure_figure.Metrics):
far, frr = bob.measure.farfrr(
dev_score[0], dev_score[1], threshold
)
dev_far_str = "%.3f%%" % (100 * far)
dev_frr_str = "%.3f%%" % (100 * frr)
dev_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
dev_far_str = "%.1f%%" % (100 * far)
dev_frr_str = "%.1f%%" % (100 * frr)
dev_mindcf_str = "%.1f%%" % ((cost * far + (1 - cost) * frr) * 100.)
raws = [['FAR', dev_far_str],
['FRR', dev_frr_str],
['minDCF', dev_mindcf_str]]
if self._test and test_score is not None:
if self._eval and eval_score is not None:
# apply threshold to development set
far, frr = bob.measure.farfrr(
test_score[0], test_score[1], threshold
eval_score[0], eval_score[1], threshold
)
test_far_str = "%.3f%%" % (100 * far)
test_frr_str = "%.3f%%" % (100 * frr)
test_mindcf_str = "%.3f%%" % ((cost * far + (1 - cost) * frr) * 100.)
raws[0].append(test_far_str)
raws[1].append(test_frr_str)
raws[2].append(test_mindcf_str)
eval_far_str = "%.1f%%" % (100 * far)
eval_frr_str = "%.1f%%" % (100 * frr)
eval_mindcf_str = "%.1f%%" % ((cost * far + (1 - cost) * frr) * 100.)
raws[0].append(eval_far_str)
raws[1].append(eval_frr_str)
raws[2].append(eval_mindcf_str)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
......@@ -183,24 +184,36 @@ class Metrics(measure_figure.Metrics):
min_cllr = bob.measure.calibration.min_cllr(
dev_score[0], dev_score[1]
)
dev_cllr_str = "%.3f%%" % cllr
dev_min_cllr_str = "%.3f%%" % min_cllr
dev_cllr_str = "%.1f%%" % cllr
dev_min_cllr_str = "%.1f%%" % min_cllr
raws = [['Cllr', dev_cllr_str],
['minCllr', dev_min_cllr_str]]
if self._test and test_score is not None:
cllr = bob.measure.calibration.cllr(test_score[0],
test_score[1])
if self._eval and eval_score is not None:
cllr = bob.measure.calibration.cllr(eval_score[0],
eval_score[1])
min_cllr = bob.measure.calibration.min_cllr(
test_score[0], test_score[1]
eval_score[0], eval_score[1]
)
test_cllr_str = "%.3f%%" % cllr
test_min_cllr_str = "%.3f%%" % min_cllr
raws[0].append(test_cllr_str)
raws[1].append(test_min_cllr_str)
eval_cllr_str = "%.1f%%" % cllr
eval_min_cllr_str = "%.1f%%" % min_cllr
raws[0].append(eval_cllr_str)
raws[1].append(eval_min_cllr_str)
click.echo(
tabulate(raws, headers, self._tablefmt), file=self.log_file
)
else:
super(Metrics, self).compute(
idx, dev_score, dev_file, test_score, test_file
idx, dev_score, dev_file, eval_score, eval_file
)
class Hist(measure_figure.Hist):
''' Histograms for biometric scores '''
def _setup_hist(self, neg, pos):
self._title_base = 'Bio scores'
self._density_hist(
pos, label='Genuines', alpha=0.9, color='C2', **self._kwargs
)
self._density_hist(
neg, label='Zero-effort impostors', alpha=0.8, color='C0', **self._kwargs
)
......@@ -11,7 +11,7 @@ def test_metrics():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
result = runner.invoke(commands.metrics, ['--no-evaluation', dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -24,7 +24,7 @@ def test_metrics():
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--test', dev1, test1, dev2, test2]
commands.metrics, [dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -32,18 +32,19 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
commands.metrics, ['-l', 'tmp', '-t', 'A,B',
dev1, test1, dev2, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--test', dev1, test2]
commands.metrics, ['-l', 'tmp', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '-t', '-T', '0.1',
commands.metrics, ['-l', 'tmp', '-T', '0.1',
'--criter', 'mindcf', '--cost', 0.9,
dev1, test2]
)
......@@ -51,7 +52,7 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp',
commands.metrics, ['--no-evaluation', '-l', 'tmp',
'--criter', 'mindcf', '--cost', 0.9,
dev1]
)
......@@ -59,27 +60,28 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-t', '--criter', 'cllr', dev1, test2]
commands.metrics, ['--criter', 'cllr', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criter', 'cllr', '--cost', 0.9,
dev1]
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'cllr',
'--cost', 0.9, dev1]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-t', '--criter', 'rr', '-T',
commands.metrics, ['--criter', 'rr', '-T',
'0.1', dev1, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criter', 'rr', dev1, dev2]
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'rr',
dev1, dev2]
)
assert result.exit_code == 0
......@@ -89,7 +91,8 @@ def test_roc():
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output','test.pdf',dev1])
result = runner.invoke(commands.roc, ['--no-evaluation', '--output',
'test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -100,7 +103,7 @@ def test_roc():
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--test', '--split', '--output',
result = runner.invoke(commands.roc, ['--split', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
......@@ -108,7 +111,7 @@ def test_roc():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--test', '--output',
result = runner.invoke(commands.roc, ['--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
......@@ -121,7 +124,7 @@ def test_det():
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.det, [dev1])
result = runner.invoke(commands.det, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -132,14 +135,14 @@ def test_det():
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--test', '--split', '--output',
result = runner.invoke(commands.det, ['--split', '--output',
'test.pdf', '--titles', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--test', '--output',
result = runner.invoke(commands.det, ['--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
......@@ -180,23 +183,24 @@ def test_hist():
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
'HISTO.pdf', '-b', 30,
dev1, dev2])
'HISTO.pdf', '-b',
30,'--no-evaluation', dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F',
3, dev1, test1, dev2, test2])
'HISTO.pdf', '-b', 30, '-F', 3,
'-t', 'A,B', dev1, test1, dev2,
test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -206,14 +210,14 @@ def test_cmc():
'data/scores-cmc-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [dev1])
result = runner.invoke(commands.cmc, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, ['--output', 'test.pdf', '-t',
result = runner.invoke(commands.cmc, ['--output', 'test.pdf',
'--titles', 'A,B', '-F', 3,
dev1, test1, dev1, test1])
if result.output:
......@@ -225,14 +229,14 @@ def test_dic():
'data/scores-nonorm-openset-dev')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.dic, [dev1, '--rank', 2])
result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.dic, ['--output', 'test.pdf', '-t',
result = runner.invoke(commands.dic, ['--output', 'test.pdf',
'--titles', 'A,B', '-F', 3,
dev1, test1, dev1, test1])
if result.output:
......@@ -251,54 +255,17 @@ def test_evaluate():
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03, '-M',
'-x', '-m', dev1, dev2])
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
'--no-evaluation', dev1, dev2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-f', 0.02, '-M',
'-x', '-m', dev1, dev2])
result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
dev1, dev2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04, '-M',
'-x', '-m', '-t', dev1, test1,
dev2, test2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-f', 0.01, '-M', '-t',
'-x', '-m', dev1, test1, dev2,
test2])
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
dev1, test1, dev2, test2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, [dev1, dev2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-R', '-D', '-H', '-E',
'-o', 'PLOTS.pdf', dev1, dev2])
result = runner.invoke(commands.evaluate, ['-f', 0.01,
dev1, test1, dev2, test2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-t', '-R', '-D', '-H', '-E',
'-o', 'PLOTS.pdf',
test1, dev1, test2, dev2])
assert result.exit_code == 0
cmc = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
cmc2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-r', cmc])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-r', '-t', cmc, cmc2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-C', '-t', cmc, cmc2])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-C', cmc, cmc2])
assert result.exit_code == 0
cmc = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col-open-set.txt')
cmc2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-O', cmc])
assert result.exit_code == 0
result = runner.invoke(commands.evaluate, ['-O', '-t', cmc, cmc2])
assert result.exit_code == 0
......@@ -93,6 +93,7 @@ Scoring
bob.bio.base.tools.compute_scores
bob.bio.base.tools.concatenate
bob.bio.base.tools.calibrate
bob.bio.base.script.figure.Metrics
Loading data
------------
......@@ -108,6 +109,13 @@ Loading data
bob.bio.base.score.load.split_five_column
bob.bio.base.score.load.cmc_five_column
Plotting
--------
.. autosummary::
bob.bio.base.script.figure.Cmc
bob.bio.base.script.figure.Dic
bob.bio.base.script.figure.Hist
OpenBR conversions
------------------
.. autosummary::
......@@ -130,4 +138,7 @@ Details
.. automodule:: bob.bio.base.score.load
.. automodule:: bob.bio.base.score.openbr
.. automodule:: bob.bio.base.script.figure
.. automodule:: bob.bio.base.script.commands
.. include:: links.rst
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment