Commit 204975b7 authored by Manuel Günther's avatar Manuel Günther
Browse files

Implemented test case and fixed some small issues in collect_result script

parent 0b58f5cd
Pipeline #13423 passed with stages
in 22 minutes and 22 seconds
......@@ -97,13 +97,13 @@ class Result:
DIR_dev = bob.measure.detection_identification_rate(scores_eval, threshold)
else:
DIR_eval = None
return DIR_dev, DIR_eval
return (DIR_dev, DIR_eval)
else:
# Recognition Rate
RR_dev = bob.measure.recognition_rate(scores_dev)
RR_eval = None if eval_file is None else bob.measure.recognition_rate(scores_eval)
return RR_dev, RR_eval
return (RR_dev, RR_eval)
else:
......@@ -140,10 +140,13 @@ class Result:
def ztnorm(self, dev_file, eval_file = None):
self.ztnorm_dev, self.ztnorm_eval = self._calculate(dev_file, eval_file)
def valid(self):
return any(a is not None for a in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval])
def __str__(self):
str = ""
for v in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]:
if v:
if v is not None:
val = "% 2.3f%%"%(v*100)
else:
val = "None"
......@@ -153,8 +156,6 @@ class Result:
return str[5:]
results = []
def add_results(args, nonorm, ztnorm = None):
"""Adds results of the given nonorm and ztnorm directories."""
r = Result(os.path.dirname(nonorm).replace(args.directory+"/", ""), args)
......@@ -178,6 +179,7 @@ def add_results(args, nonorm, ztnorm = None):
else:
r.ztnorm(dev_file)
global results
results.append(r)
......@@ -186,8 +188,8 @@ def recurse(args, path):
dir_list = os.listdir(path)
# check if the score directories are included in the current path
if args.nonorm in dir_list:
if args.ztnorm in dir_list:
if args.nonorm in dir_list or args.nonorm == '.':
if args.ztnorm in dir_list or args.ztnorm == '.':
add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm))
else:
add_results(args, os.path.join(path, args.nonorm))
......@@ -203,7 +205,8 @@ def table():
A = " "*2 + 'dev nonorm'+ " "*5 + 'dev ztnorm' + " "*6 + 'eval nonorm' + " "*4 + 'eval ztnorm' + " "*12 + 'directory\n'
A += "-"*100+"\n"
for r in results:
A += str(r) + "\n"
if r.valid():
A += str(r) + "\n"
return A
......@@ -211,6 +214,8 @@ def main(command_line_parameters = None):
"""Iterates through the desired directory and collects all result files."""
args = command_line_arguments(command_line_parameters)
global results
results = []
# collect results
directories = glob.glob(args.directory)
for directory in directories:
......
......@@ -537,7 +537,7 @@ def test_evaluate_openset():
def test_resources():
# simply test that the collect_results script works
# simply test that the resorces script works
from bob.bio.base.script.resources import resources, databases
with utils.Quiet():
resources(['--types', 'database', 'preprocessor', 'extractor', 'algorithm', 'grid', '--details', '--packages', 'bob.bio.base'])
......@@ -546,13 +546,36 @@ def test_resources():
def test_collect_results():
# simply test that the collect_results script works
test_dir = tempfile.mkdtemp(prefix='bobtest_')
try:
from bob.bio.base.script.collect_results import main
main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test'])
finally:
if os.path.exists(test_dir):
os.rmdir(test_dir)
from bob.bio.base.script.collect_results import main
# FAR criterion
main([
'-D', data_dir,
'-d', 'scores-nonorm-dev',
'-e', 'scores-nonorm-fivecol-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir', '--criterion', 'FAR',
'--self-test', '-vvv'
])
# Recognition Rate
main([
'-D', data_dir,
'-d', 'scores-nonorm-dev',
'-e', 'scores-nonorm-fivecol-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir', '--criterion', 'RR',
'--self-test', '-vvv'
])
# DIR
main([
'-D', data_dir,
'-d', 'scores-nonorm-openset-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir', '--criterion', 'DIR',
'--self-test', '-vvv'
])
@utils.grid_available
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment