Commit cd839a7e authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Rewrite old evaluate tests and remove them

parent 1deb6d67
Pipeline #19902 failed with stage
in 29 minutes and 31 seconds
......@@ -243,7 +243,7 @@ def cmc(ctx, scores, evaluation, **kargs):
@common_options.title_option()
@common_options.legends_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='cmc.pdf')
@common_options.output_plot_file_option(default_out='dir.pdf')
@common_options.eval_option()
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
......@@ -253,6 +253,7 @@ def cmc(ctx, scores, evaluation, **kargs):
@common_options.style_option()
@common_options.linestyles_option()
@common_options.figsize_option()
@common_options.min_far_option()
@verbosity_option()
@click.pass_context
def dir(ctx, scores, evaluation, **kargs):
......
'''Plots and measures for bob.bio.base'''
import math
import click
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
......@@ -76,6 +77,8 @@ class Dir(measure_figure.PlotBase):
self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR'
def compute(self, idx, input_scores, input_names):
''' Plot DIR for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`'''
......@@ -104,6 +107,9 @@ class Dir(measure_figure.PlotBase):
label=self._label('development', input_names[0], idx)
)
if self._min_dig is not None:
mpl.xlim(xmin=math.pow(10, self._min_dig))
class Metrics(measure_figure.Metrics):
''' Compute metrics from score files'''
def init_process(self):
......
......@@ -86,6 +86,7 @@ def test_metrics():
assert result.exit_code == 0, (result.exit_code, result.output)
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
......@@ -119,6 +120,22 @@ def test_roc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, [
'--min-far-value', '1e-6',
'--lines-at', '1e-5',
'-v', '--legends', 'A',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_det():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -150,6 +167,23 @@ def test_det():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.det, [
'--min-far-value', '1e-6',
'--lines-at', '1e-5',
'-v', '--legends', 'A',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
......@@ -173,6 +207,22 @@ def test_epc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, [
'-v', '--legends', 'A',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
......@@ -206,6 +256,8 @@ def test_hist():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-5col.txt')
......@@ -225,6 +277,23 @@ def test_cmc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [
'-v', '--legends', 'A',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_dir():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
......@@ -239,34 +308,8 @@ def test_dir():
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
'--min-far-value', '1e-6',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
'--no-evaluation', dev1, dev2])
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
dev1, dev2])
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
dev1, test1, dev2, test2])
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-f', 0.01,
dev1, test1, dev2, test2])
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -491,67 +491,6 @@ def test_fusion():
def test_evaluate_closedset():
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det', 'epc']]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
'--directory', data_dir, # will not be ignored since reference files are relative
'--legends', 'no norm', 'ZT norm',
'--criterion', 'HTER',
'--roc', plots[0],
'--cmc', plots[1],
'--det', plots[2],
'--epc', plots[3],
'--rr',
'--thresholds', '5000', '0',
'--min-far-value', '1e-6',
'--far-line-at', '1e-5',
'-v',
]
# execute the script
from bob.bio.base.script.evaluate import main
try:
main(parameters)
for i in range(4):
assert os.path.exists(plots[i])
os.remove(plots[i])
finally:
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
def test_evaluate_openset():
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_file = os.path.join(data_dir, 'scores-nonorm-openset-dev')
plot = os.path.join(test_dir, 'dir.pdf')
parameters = [
'--dev-files', reference_file,
'--eval-files', reference_file,
'--directory', "/non/existing/directory", # will be ignored since reference_file is absolute
'--legends', 'Test',
'--dir', plot,
'--min-far-value', '1e-6',
'-v',
]
# execute the script
from bob.bio.base.script.evaluate import main
try:
main(parameters)
assert os.path.exists(plot)
os.remove(plot)
finally:
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
def test_resources():
# simply test that the resorces script works
from bob.bio.base.script.resources import resources, databases
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment