Commit c9789352 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Add measure scripts: metrics (HTER, EER), roc, det, epc, hist (score...

Add measure scripts: metrics (HTER, EER), roc, det, epc, hist (score histograms) and evaluate (that generates all the previous at once). Scripts can be called using  with xxx = roc for example. Add tests for those scripts.
parent cb12cf37
Pipeline #17854 failed with stage
in 31 minutes and 19 seconds
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
[Min. criterion: EER] Threshold on Development set `dev-1.txt`: -8.025286e-03
╒══════╤═════════════════════════╕
│ │ Development dev-1.txt │
╞══════╪═════════════════════════╡
│ FMR │ 6.263% (31/495) │
├──────┼─────────────────────────┤
│ FNMR │ 6.208% (28/451) │
├──────┼─────────────────────────┤
│ FAR │ 5.924% │
├──────┼─────────────────────────┤
│ FRR │ 11.273% │
├──────┼─────────────────────────┤
│ HTER │ 8.599% │
╘══════╧═════════════════════════╛
[Min. criterion: EER] Threshold on Development set `dev-1.txt`: -8.025286e-03
╒══════╤═════════════════════════╤═══════════════════╕
│ │ Development dev-1.txt │ Test test-1.txt │
╞══════╪═════════════════════════╪═══════════════════╡
│ FMR │ 6.263% (31/495) │ 5.637% (27/479) │
├──────┼─────────────────────────┼───────────────────┤
│ FNMR │ 6.208% (28/451) │ 6.131% (29/473) │
├──────┼─────────────────────────┼───────────────────┤
│ FAR │ 5.924% │ 5.366% │
├──────┼─────────────────────────┼───────────────────┤
│ FRR │ 11.273% │ 10.637% │
├──────┼─────────────────────────┼───────────────────┤
│ HTER │ 8.599% │ 8.001% │
╘══════╧═════════════════════════╧═══════════════════╛
[Min. criterion: EER] Threshold on Development set `dev-2.txt`: 1.652567e-02
╒══════╤═════════════════════════╤═══════════════════╕
│ │ Development dev-2.txt │ Test test-2.txt │
╞══════╪═════════════════════════╪═══════════════════╡
│ FMR │ 4.591% (23/501) │ 3.333% (16/480) │
├──────┼─────────────────────────┼───────────────────┤
│ FNMR │ 4.484% (20/446) │ 7.006% (33/471) │
├──────┼─────────────────────────┼───────────────────┤
│ FAR │ 4.348% │ 3.170% │
├──────┼─────────────────────────┼───────────────────┤
│ FRR │ 9.547% │ 11.563% │
├──────┼─────────────────────────┼───────────────────┤
│ HTER │ 6.947% │ 7.367% │
╘══════╧═════════════════════════╧═══════════════════╛
This diff is collapsed.
This diff is collapsed.
......@@ -3,10 +3,12 @@
import click
import pkg_resources
from click_plugins import with_plugins
from click.types import INT, FLOAT, Choice, File
@with_plugins(pkg_resources.iter_entry_points('bob.measure.cli'))
@click.group()
@click.group(chain=True)
def measure():
"""Entry for bob.measure commands."""
pass
'''Tests for bob.measure scripts'''
import os
import filecmp
import tempfile
import subprocess
import click
from click.testing import CliRunner
import bob.io.base.test_utils
from .script import evaluate
def test_metrics():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
result = runner.invoke(evaluate.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
assert filecmp.cmp(test_ref, 'tmp')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['--test', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m2.txt', 'bob.measure')
assert filecmp.cmp(test_ref, 'tmp')
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, test1, dev2, test2]
)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.metrics, ['-l', 'tmp', '--test', dev1, dev2]
)
assert result.exit_code == 0
def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--output','test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.roc, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_det():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.det, ['--test', '--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_epc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(evaluate.epc, ['--output', 'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_hist():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, ['--criter', 'hter','--output',
'HISTO.pdf', '-b', 30,
dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(evaluate.hist, ['--criter', 'eer','--output',
'HISTO.pdf', '-b', 30,
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
def test_evaluate():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(evaluate.evaluate, [dev1])
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.evaluate, ['--output', 'my_plots.pdf', '-b', 30,
'-n', 300, dev1, dev2])
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
evaluate.evaluate, ['-t', dev1, test1, dev2, test2])
assert result.exit_code == 0
......@@ -73,6 +73,11 @@ setup(
# bob measure scripts
'bob.measure.cli': [
'evaluate = bob.measure.script.evaluate:evaluate',
'metrics = bob.measure.script.evaluate:metrics',
'roc = bob.measure.script.evaluate:roc',
'det = bob.measure.script.evaluate:det',
'epc = bob.measure.script.evaluate:epc',
'hist = bob.measure.script.evaluate:hist',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment