Commit bde121aa authored by Manuel Günther's avatar Manuel Günther
Browse files

Implemented five-column score file -- only during concatenation

parent 57bb63d9
Pipeline #11286 passed with stages
in 20 minutes and 37 seconds
......@@ -368,7 +368,8 @@ def execute(args):
tools.concatenate(
args.zt_norm,
groups = [args.group],
write_compressed = args.write_compressed_score_files)
write_compressed = args.write_compressed_score_files,
add_model_id = args.write_five_column_score_files)
# calibrate scores
elif args.sub_task == 'calibrate':
......
This diff is collapsed.
This diff is collapsed.
......@@ -21,7 +21,7 @@ regenerate_reference = False
dummy_dir = pkg_resources.resource_filename('bob.bio.base', 'test/dummy')
data_dir = pkg_resources.resource_filename('bob.bio.base', 'test/data')
def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores','')):
def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores',''), counts=3):
from bob.bio.base.script.verify import main
try:
main(parameters)
......@@ -51,9 +51,9 @@ def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('sco
assert d[0].shape == d[1].shape
# assert that the data order is still correct
assert (d[0][:,0:3] == d[1][:, 0:3]).all()
assert (d[0][:,0:counts] == d[1][:, 0:counts]).all()
# assert that the values are OK
assert numpy.allclose(d[0][:,3].astype(float), d[1][:,3].astype(float), 1e-5)
assert numpy.allclose(d[0][:,counts].astype(float), d[1][:,counts].astype(float), 1e-5)
assert not os.path.exists(os.path.join(test_dir, 'submitted.sql3'))
......@@ -295,6 +295,25 @@ def test_verify_missing():
shutil.rmtree(test_dir)
def test_verify_five_col():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'dummy',
'-e', 'dummy',
'-a', 'dummy',
'--zt-norm',
'--write-five-column-score-files',
'-vs', 'test_missing',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.base',
'--imports', 'bob.bio.base.test.dummy'
]
_verify(parameters, test_dir, 'test_missing', ref_modifier="-fivecol", counts=4)
def test_verify_execute_only():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
......
......@@ -171,6 +171,8 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
help='Only report the commands that will be executed, but do not execute them.')
flag_group.add_argument('-F', '--force', action='store_true',
help='Force to erase former data if already exist')
flag_group.add_argument('-U', '--write-five-column-score-files', action='store_true',
help='Writes score files in five-column format (including the model id)')
flag_group.add_argument('-Z', '--write-compressed-score-files', action='store_true',
help='Writes score files which are compressed with tar.bz2.')
flag_group.add_argument('-S', '--stop-on-failure', action='store_true',
......
......@@ -442,20 +442,28 @@ def zt_norm(groups = ['dev', 'eval'], write_compressed = False, allow_missing_fi
def _concat(score_files, output, write_compressed):
def _concat(score_files, output, write_compressed, model_ids):
"""Concatenates a list of score files into a single score file."""
f = _open_to_write(output, write_compressed)
# Concatenates the scores
for score_file in score_files:
i = _open_to_read(score_file)
f.write(i.read())
if model_ids is None:
for score_file in score_files:
i = _open_to_read(score_file)
f.write(i.read())
else:
for score_file, model_id in zip(score_files, model_ids):
i = _open_to_read(score_file)
for l in i:
s = l.split()
s.insert(1, str(model_id))
f.write(" ".join(s) + "\n")
_close_written(output, f, write_compressed)
def concatenate(compute_zt_norm, groups = ['dev', 'eval'], write_compressed = False):
def concatenate(compute_zt_norm, groups = ['dev', 'eval'], write_compressed = False, add_model_id = False):
"""Concatenates all results into one (or two) score files per group.
Score files, which were generated per model, are concatenated into a single score file, which can be interpreter by :py:func:`bob.measure.load.split_four_column`.
......@@ -477,15 +485,16 @@ def concatenate(compute_zt_norm, groups = ['dev', 'eval'], write_compressed = Fa
for group in groups:
logger.info("- Scoring: concatenating score files for group '%s'", group)
# (sorted) list of models
model_files = [fs.no_norm_file(model_id, group) for model_id in fs.model_ids(group)]
model_ids = fs.model_ids(group)
model_files = [fs.no_norm_file(model_id, group) for model_id in model_ids]
result_file = fs.no_norm_result_file(group)
_concat(model_files, result_file, write_compressed)
_concat(model_files, result_file, write_compressed, model_ids if add_model_id else None)
logger.info("- Scoring: wrote score file '%s'", result_file)
if compute_zt_norm:
model_files = [fs.zt_norm_file(model_id, group) for model_id in fs.model_ids(group)]
model_files = [fs.zt_norm_file(model_id, group) for model_id in model_ids]
result_file = fs.zt_norm_result_file(group)
_concat(model_files, result_file, write_compressed)
_concat(model_files, result_file, write_compressed, model_ids if add_model_id else None)
logger.info("- Scoring: wrote score file '%s'", result_file)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment