diff --git a/bob/bio/base/script/collect_results.py b/bob/bio/base/script/collect_results.py index b4d9ed04c9a7b33a36fe8cac754da12f0743b925..3c549d79976f092ceea4a6686e81bb87868f2113 100644 --- a/bob/bio/base/script/collect_results.py +++ b/bob/bio/base/script/collect_results.py @@ -97,13 +97,13 @@ class Result: DIR_dev = bob.measure.detection_identification_rate(scores_eval, threshold) else: DIR_eval = None - return DIR_dev, DIR_eval + return (DIR_dev, DIR_eval) else: # Recognition Rate RR_dev = bob.measure.recognition_rate(scores_dev) RR_eval = None if eval_file is None else bob.measure.recognition_rate(scores_eval) - return RR_dev, RR_eval + return (RR_dev, RR_eval) else: @@ -140,10 +140,13 @@ class Result: def ztnorm(self, dev_file, eval_file = None): self.ztnorm_dev, self.ztnorm_eval = self._calculate(dev_file, eval_file) + def valid(self): + return any(a is not None for a in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]) + def __str__(self): str = "" for v in [self.nonorm_dev, self.ztnorm_dev, self.nonorm_eval, self.ztnorm_eval]: - if v: + if v is not None: val = "% 2.3f%%"%(v*100) else: val = "None" @@ -153,8 +156,6 @@ class Result: return str[5:] -results = [] - def add_results(args, nonorm, ztnorm = None): """Adds results of the given nonorm and ztnorm directories.""" r = Result(os.path.dirname(nonorm).replace(args.directory+"/", ""), args) @@ -178,6 +179,7 @@ def add_results(args, nonorm, ztnorm = None): else: r.ztnorm(dev_file) + global results results.append(r) @@ -186,8 +188,8 @@ def recurse(args, path): dir_list = os.listdir(path) # check if the score directories are included in the current path - if args.nonorm in dir_list: - if args.ztnorm in dir_list: + if args.nonorm in dir_list or args.nonorm == '.': + if args.ztnorm in dir_list or args.ztnorm == '.': add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm)) else: add_results(args, os.path.join(path, args.nonorm)) @@ -203,7 +205,8 @@ def table(): A = " "*2 + 'dev nonorm'+ " "*5 + 'dev ztnorm' + " "*6 + 'eval nonorm' + " "*4 + 'eval ztnorm' + " "*12 + 'directory\n' A += "-"*100+"\n" for r in results: - A += str(r) + "\n" + if r.valid(): + A += str(r) + "\n" return A @@ -211,6 +214,8 @@ def main(command_line_parameters = None): """Iterates through the desired directory and collects all result files.""" args = command_line_arguments(command_line_parameters) + global results + results = [] # collect results directories = glob.glob(args.directory) for directory in directories: diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py index 0226f6de17bba1a9cf449d48801b316f651995dd..9072b717a5f20de906d027f45fd644086763a7dc 100644 --- a/bob/bio/base/test/test_scripts.py +++ b/bob/bio/base/test/test_scripts.py @@ -537,7 +537,7 @@ def test_evaluate_openset(): def test_resources(): - # simply test that the collect_results script works + # simply test that the resorces script works from bob.bio.base.script.resources import resources, databases with utils.Quiet(): resources(['--types', 'database', 'preprocessor', 'extractor', 'algorithm', 'grid', '--details', '--packages', 'bob.bio.base']) @@ -546,13 +546,36 @@ def test_resources(): def test_collect_results(): # simply test that the collect_results script works - test_dir = tempfile.mkdtemp(prefix='bobtest_') - try: - from bob.bio.base.script.collect_results import main - main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test']) - finally: - if os.path.exists(test_dir): - os.rmdir(test_dir) + from bob.bio.base.script.collect_results import main + # FAR criterion + main([ + '-D', data_dir, + '-d', 'scores-nonorm-dev', + '-e', 'scores-nonorm-fivecol-dev', + '-n', '.', '-z', '.', + '--sort', '--sort-key', 'dir', '--criterion', 'FAR', + '--self-test', '-vvv' + ]) + + # Recognition Rate + main([ + '-D', data_dir, + '-d', 'scores-nonorm-dev', + '-e', 'scores-nonorm-fivecol-dev', + '-n', '.', '-z', '.', + '--sort', '--sort-key', 'dir', '--criterion', 'RR', + '--self-test', '-vvv' + ]) + + # DIR + main([ + '-D', data_dir, + '-d', 'scores-nonorm-openset-dev', + '-n', '.', '-z', '.', + '--sort', '--sort-key', 'dir', '--criterion', 'DIR', + '--self-test', '-vvv' + ]) + @utils.grid_available