From cd839a7e1da9d0662436b5cb346e79e49b5617b8 Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Mon, 7 May 2018 14:49:12 +0200
Subject: [PATCH] Rewrite old evaluate tests and remove them
---
bob/bio/base/script/commands.py | 3 +-
bob/bio/base/script/figure.py | 6 ++
bob/bio/base/test/test_commands.py | 97 +++++++++++++++++++++---------
bob/bio/base/test/test_scripts.py | 61 -------------------
4 files changed, 78 insertions(+), 89 deletions(-)
diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
index e4d4ce6b..f712823e 100644
--- a/bob/bio/base/script/commands.py
+++ b/bob/bio/base/script/commands.py
@@ -243,7 +243,7 @@ def cmc(ctx, scores, evaluation, **kargs):
@common_options.title_option()
@common_options.legends_option()
@common_options.sep_dev_eval_option()
-@common_options.output_plot_file_option(default_out='cmc.pdf')
+@common_options.output_plot_file_option(default_out='dir.pdf')
@common_options.eval_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
@@ -253,6 +253,7 @@ def cmc(ctx, scores, evaluation, **kargs):
@common_options.style_option()
@common_options.linestyles_option()
@common_options.figsize_option()
+@common_options.min_far_option()
@verbosity_option()
@click.pass_context
def dir(ctx, scores, evaluation, **kargs):
diff --git a/bob/bio/base/script/figure.py b/bob/bio/base/script/figure.py
index 00242f77..ec95d88e 100644
--- a/bob/bio/base/script/figure.py
+++ b/bob/bio/base/script/figure.py
@@ -1,5 +1,6 @@
'''Plots and measures for bob.bio.base'''
+import math
import click
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
@@ -76,6 +77,8 @@ class Dir(measure_figure.PlotBase):
self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR'
+
+
def compute(self, idx, input_scores, input_names):
''' Plot DIR for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`'''
@@ -104,6 +107,9 @@ class Dir(measure_figure.PlotBase):
label=self._label('development', input_names[0], idx)
)
+ if self._min_dig is not None:
+ mpl.xlim(xmin=math.pow(10, self._min_dig))
+
class Metrics(measure_figure.Metrics):
''' Compute metrics from score files'''
def init_process(self):
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
index 9d3dbb25..7ed0a6a6 100644
--- a/bob/bio/base/test/test_commands.py
+++ b/bob/bio/base/test/test_commands.py
@@ -86,6 +86,7 @@ def test_metrics():
assert result.exit_code == 0, (result.exit_code, result.output)
+
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
@@ -119,6 +120,22 @@ def test_roc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
+ dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-nonorm-dev')
+ dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-ztnorm-dev')
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.roc, [
+ '--min-far-value', '1e-6',
+ '--lines-at', '1e-5',
+ '-v', '--legends', 'A',
+ dev_nonorm, dev_ztnorm
+ ])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
+
def test_det():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -150,6 +167,23 @@ def test_det():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
+
+ dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-nonorm-dev')
+ dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-ztnorm-dev')
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.det, [
+ '--min-far-value', '1e-6',
+ '--lines-at', '1e-5',
+ '-v', '--legends', 'A',
+ dev_nonorm, dev_ztnorm
+ ])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
+
def test_epc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
@@ -173,6 +207,22 @@ def test_epc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
+ dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-nonorm-dev')
+ dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-ztnorm-dev')
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.epc, [
+ '-v', '--legends', 'A',
+ dev_nonorm, dev_ztnorm
+ ])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
+
+
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
@@ -206,6 +256,8 @@ def test_hist():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
+
+
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-5col.txt')
@@ -225,6 +277,23 @@ def test_cmc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
+ dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-nonorm-dev')
+ dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
+ 'data/scores-ztnorm-dev')
+
+ with runner.isolated_filesystem():
+ result = runner.invoke(commands.cmc, [
+ '-v', '--legends', 'A',
+ dev_nonorm, dev_ztnorm
+ ])
+ if result.output:
+ click.echo(result.output)
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
+
+
+
def test_dir():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
@@ -239,34 +308,8 @@ def test_dir():
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
+ '--min-far-value', '1e-6',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
-
-def test_evaluate():
- dev1 = pkg_resources.resource_filename('bob.bio.base.test',
- 'data/dev-4col.txt')
- dev2 = pkg_resources.resource_filename('bob.bio.base.test',
- 'data/dev-5col.txt')
-
- test1 = pkg_resources.resource_filename('bob.bio.base.test',
- 'data/test-4col.txt')
- test2 = pkg_resources.resource_filename('bob.bio.base.test',
- 'data/test-5col.txt')
- runner = CliRunner()
- with runner.isolated_filesystem():
- result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
- '--no-evaluation', dev1, dev2])
- assert result.exit_code == 0, (result.exit_code, result.output)
- result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
- dev1, dev2])
- assert result.exit_code == 0, (result.exit_code, result.output)
-
- result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
- dev1, test1, dev2, test2])
- assert result.exit_code == 0, (result.exit_code, result.output)
- result = runner.invoke(commands.evaluate, ['-f', 0.01,
- dev1, test1, dev2, test2])
- assert result.exit_code == 0, (result.exit_code, result.output)
-
diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py
index 912dd325..582739bf 100644
--- a/bob/bio/base/test/test_scripts.py
+++ b/bob/bio/base/test/test_scripts.py
@@ -491,67 +491,6 @@ def test_fusion():
-def test_evaluate_closedset():
- # tests our 'evaluate' script using the reference files
- test_dir = tempfile.mkdtemp(prefix='bobtest_')
- reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
- plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det', 'epc']]
- parameters = [
- '--dev-files', reference_files[0], reference_files[1],
- '--eval-files', reference_files[0], reference_files[1],
- '--directory', data_dir, # will not be ignored since reference files are relative
- '--legends', 'no norm', 'ZT norm',
- '--criterion', 'HTER',
- '--roc', plots[0],
- '--cmc', plots[1],
- '--det', plots[2],
- '--epc', plots[3],
- '--rr',
- '--thresholds', '5000', '0',
- '--min-far-value', '1e-6',
- '--far-line-at', '1e-5',
- '-v',
- ]
-
- # execute the script
- from bob.bio.base.script.evaluate import main
- try:
- main(parameters)
- for i in range(4):
- assert os.path.exists(plots[i])
- os.remove(plots[i])
- finally:
- if os.path.exists(test_dir):
- shutil.rmtree(test_dir)
-
-def test_evaluate_openset():
- # tests our 'evaluate' script using the reference files
- test_dir = tempfile.mkdtemp(prefix='bobtest_')
- reference_file = os.path.join(data_dir, 'scores-nonorm-openset-dev')
- plot = os.path.join(test_dir, 'dir.pdf')
- parameters = [
- '--dev-files', reference_file,
- '--eval-files', reference_file,
- '--directory', "/non/existing/directory", # will be ignored since reference_file is absolute
- '--legends', 'Test',
- '--dir', plot,
- '--min-far-value', '1e-6',
- '-v',
- ]
-
- # execute the script
- from bob.bio.base.script.evaluate import main
- try:
- main(parameters)
- assert os.path.exists(plot)
- os.remove(plot)
- finally:
- if os.path.exists(test_dir):
- shutil.rmtree(test_dir)
-
-
-
-
def test_resources():
# simply test that the resorces script works
from bob.bio.base.script.resources import resources, databases
--
GitLab