diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
index 397120850a2d6248063fb191c427346f49482c7c..0b59d3232f447da717a0262fcd27af55f4cf7b64 100644
--- a/bob/bio/base/script/commands.py
+++ b/bob/bio/base/script/commands.py
@@ -41,8 +41,8 @@ def metrics(ctx, scores, evaluation, **kargs):
criterion (eer, min-hter, far, mindcf, cllr, rr).
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -54,11 +54,11 @@ def metrics(ctx, scores, evaluation, **kargs):
Examples:
$ bob bio metrics dev-scores
- $ bob bio metrics --no-evaluation dev-scores1 dev-scores2
+ $ bob bio metrics dev-scores1 dev-scores2
- $ bob bio metrics -l results.txt dev-scores1 eval-scores1
+ $ bob bio metrics -e -l results.txt dev-scores1 eval-scores1
- $ bob bio metrics {dev,eval}-scores1 {dev,eval}-scores2
+ $ bob bio metrics -e {dev,eval}-scores1 {dev,eval}-scores2
"""
if 'criterion' in ctx.meta and ctx.meta['criterion'] == 'rr':
process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc)
@@ -97,8 +97,8 @@ def roc(ctx, scores, evaluation, **kargs):
computed using :py:func:`bob.measure.roc`.
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -106,10 +106,10 @@ def roc(ctx, scores, evaluation, **kargs):
Examples:
$ bob bio roc -v dev-scores
- $ bob bio roc -v dev-scores1 eval-scores1 dev-scores2
+ $ bob bio roc -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
- $ bob bio roc -v -o my_roc.pdf dev-scores1 eval-scores1
+ $ bob bio roc -e -v -o my_roc.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Roc(ctx, scores, evaluation, load.split)
process.run()
@@ -143,8 +143,8 @@ def det(ctx, scores, evaluation, **kargs):
(false positives on the x-axis and false negatives on the y-axis)
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -152,10 +152,10 @@ def det(ctx, scores, evaluation, **kargs):
Examples:
$ bob bio det -v dev-scores
- $ bob bio det -v dev-scores1 eval-scores1 dev-scores2
+ $ bob bio det -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
- $ bob bio det -v -o my_det.pdf dev-scores1 eval-scores1
+ $ bob bio det -e -v -o my_det.pdf dev-scores1 eval-scores1
"""
process = bio_figure.Det(ctx, scores, evaluation, load.split)
process.run()
@@ -219,8 +219,8 @@ def cmc(ctx, scores, evaluation, **kargs):
using :py:func:`bob.measure.cmc`.
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -273,14 +273,14 @@ def dir(ctx, scores, evaluation, **kargs):
.. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
- $ bob bio dir -v dev-scores
+ $ bob bio dir -e -v dev-scores
$ bob bio dir -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
@@ -297,6 +297,7 @@ def dir(ctx, scores, evaluation, **kargs):
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
+@common_options.no_line_option()
@common_options.far_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@@ -313,8 +314,8 @@ def hist(ctx, scores, evaluation, **kwargs):
criterion.
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -326,10 +327,10 @@ def hist(ctx, scores, evaluation, **kwargs):
Examples:
$ bob bio hist -v dev-scores
- $ bob bio hist -v dev-scores1 eval-scores1 dev-scores2
+ $ bob bio hist -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
- $ bob bio hist -v --criterion min-hter dev-scores1 eval-scores1
+ $ bob bio hist -e -v --criterion min-hter dev-scores1 eval-scores1
"""
process = bio_figure.Hist(ctx, scores, evaluation, load.split)
process.run()
@@ -367,8 +368,8 @@ def evaluate(ctx, scores, evaluation, **kwargs):
curves to a multi-page PDF file
You need to provide one or more development score file(s) for each
- experiment. You can also provide eval files along with dev files. If only
- dev-scores are used, the flag `--no-evaluation` must be used. is required
+ experiment. You can also provide eval files along with dev files. If
+ eval-scores are used, the flag `--eval` must be used. is required
in that case. Files must be 4- or 5- columns format, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
@@ -382,9 +383,9 @@ def evaluate(ctx, scores, evaluation, **kwargs):
Examples:
$ bob bio evaluate -v dev-scores
- $ bob bio evaluate -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
+ $ bob bio evaluate -e -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
- $ bob bio evaluate -v -o my_plots.pdf /path/to/syst-{1,2,3}/{dev,eval}-scores
+ $ bob bio evaluate -e -v -o my_plots.pdf /path/to/syst-{1,2,3}/{dev,eval}-scores
'''
log_str = ''
if 'log' in ctx.meta and ctx.meta['log'] is not None:
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
index e705e7fee0f780698b445829d87e14186c90d943..acc99ff555d0f81f596d9a7f0b7d3b531e5e58b0 100644
--- a/bob/bio/base/test/test_commands.py
+++ b/bob/bio/base/test/test_commands.py
@@ -11,7 +11,7 @@ def test_metrics():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
- result = runner.invoke(commands.metrics, ['--no-evaluation', dev1])
+ result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
@@ -24,7 +24,7 @@ def test_metrics():
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, [dev1, test1, dev2, test2]
+ commands.metrics, ['-e', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
@@ -32,19 +32,19 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', '-lg', 'A,B',
+ commands.metrics, ['-e', '-l', 'tmp', '-lg', 'A,B',
dev1, test1, dev2, test2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', dev1, test2]
+ commands.metrics, ['-e', '-l', 'tmp', dev1, test2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', '-T', '0.1',
+ commands.metrics, ['-e', '-l', 'tmp', '-T', '0.1',
'--criterion', 'mindcf', '--cost', 0.9,
dev1, test2]
)
@@ -52,7 +52,7 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['--no-evaluation', '-l', 'tmp',
+ commands.metrics, ['-l', 'tmp',
'--criterion', 'mindcf', '--cost', 0.9,
dev1]
)
@@ -60,27 +60,27 @@ def test_metrics():
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['--criterion', 'cllr', dev1, test2]
+ commands.metrics, ['-e', '--criterion', 'cllr', dev1, test2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criterion', 'cllr',
+ commands.metrics, ['-l', 'tmp', '--criterion', 'cllr',
'--cost', 0.9, dev1]
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['--criterion', 'rr', '-T',
+ commands.metrics, ['-e', '--criterion', 'rr', '-T',
'0.1', dev1, test2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criterion', 'rr',
+ commands.metrics, ['-l', 'tmp', '--criterion', 'rr',
dev1, dev2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
@@ -92,7 +92,7 @@ def test_roc():
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.roc, ['--no-evaluation', '--output',
+ result = runner.invoke(commands.roc, ['--output',
'test.pdf',dev1])
if result.output:
click.echo(result.output)
@@ -106,7 +106,7 @@ def test_roc():
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--split', '--output',
'test.pdf', '-S', '-lc',
- 'lower-left',
+ 'lower-left', '-e',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
@@ -114,7 +114,8 @@ def test_roc():
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
- 'test.pdf', '--legends', 'A,B',
+ 'test.pdf',
+ '-e', '--legends', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
@@ -128,7 +129,7 @@ def test_roc():
result = runner.invoke(commands.roc, [
'--min-far-value', '1e-6',
'--lines-at', '1e-5',
- '-v', '--legends', 'A',
+ '-v', '--legends', 'A', '-e',
dev_nonorm, dev_ztnorm
])
if result.output:
@@ -142,7 +143,7 @@ def test_det():
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.det, ['--no-evaluation', dev1, '-S'])
+ result = runner.invoke(commands.det, [dev1, '-S'])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
@@ -154,14 +155,15 @@ def test_det():
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--split', '--output',
- 'test.pdf', '--legends', 'A,B',
+ 'test.pdf', '--legends',
+ 'A,B', '-e',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
- 'test.pdf',
+ 'test.pdf', '-e',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
@@ -175,7 +177,7 @@ def test_det():
with runner.isolated_filesystem():
result = runner.invoke(commands.det, [
'--min-far-value', '1e-6',
- '--lines-at', '1e-5',
+ '--lines-at', '1e-5', '-e',
'-v', '--legends', 'A',
dev_nonorm, dev_ztnorm
])
@@ -234,7 +236,7 @@ def test_hist():
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
+ result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
@@ -242,14 +244,14 @@ def test_hist():
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion', 'min-hter', '--output',
'HISTO.pdf', '-b',
- '30,auto','--no-evaluation', dev1, dev2])
+ '30,auto', dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion', 'eer', '--output',
- 'HISTO.pdf', '-b', '30',
+ 'HISTO.pdf', '-b', '30', '-e',
'-lg', 'A,B', dev1, test1, dev2,
test2])
if result.output:
@@ -263,7 +265,7 @@ def test_cmc():
'data/scores-cmc-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.cmc, ['--no-evaluation', dev1])
+ result = runner.invoke(commands.cmc, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
@@ -272,7 +274,7 @@ def test_cmc():
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
- '-ts', 'TA,TB',
+ '-ts', 'TA,TB', '-e',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
@@ -285,7 +287,7 @@ def test_cmc():
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [
- '-v', '--legends', 'A',
+ '-v', '--legends', 'A', '-e',
dev_nonorm, dev_ztnorm
])
if result.output:
@@ -300,7 +302,7 @@ def test_dir():
'data/scores-nonorm-openset-dev')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.dir, ['--no-evaluation', dev1, '--rank', 2])
+ result = runner.invoke(commands.dir, [dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
@@ -309,7 +311,7 @@ def test_dir():
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
- '--min-far-value', '1e-6',
+ '--min-far-value', '1e-6', '-e',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
diff --git a/doc/experiments.rst b/doc/experiments.rst
index 21160bdcca5abe098e33f7ce030f04bae93c4121..863d96a7d0c447234ca459d431531f644108ff6e 100644
--- a/doc/experiments.rst
+++ b/doc/experiments.rst
@@ -177,7 +177,7 @@ min.HTER) on a development set and apply it on an evaluation set, just do:
.. code-block:: sh
- $ bob bio metrics {dev,test}-4col.txt --legends ExpA --criterion min-hter
+ $ bob bio metrics -e {dev,test}-4col.txt --legends ExpA --criterion min-hter
[Min. criterion: MIN-HTER ] Threshold on Development set `ExpA`: -4.830500e-03
====== ====================== =================
@@ -192,8 +192,8 @@ min.HTER) on a development set and apply it on an evaluation set, just do:
====== ====================== =================
.. note::
- You can compute analysis on development set(s) only by passing option
- ``--no-evaluation``. See metrics --help for further options.
+ When evaluation scores are provided, ``--eval`` option must be passed.
+ See metrics --help for further options.
You can also compute measure such as recognition rate (``rr``), Cllr and
minCllr (``cllr``) and minDCF (``mindcf``) by passing the corresponding option.
@@ -201,7 +201,7 @@ For example:
.. code-block:: sh
- bob bio metrics {dev,test}-4col.txt --legends ExpA --criterion cllr
+ bob bio metrics -e {dev,test}-4col.txt --legends ExpA --criterion cllr
====== ====================== ================
Computing Cllr and minCllr...
@@ -244,7 +244,7 @@ For example, to generate a CMC curve from development and evaluation datasets:
.. code-block:: sh
- $bob bio cmc -v --output 'my_cmc.pdf' dev-1.txt eval-1.txt
+ $bob bio cmc -e -v --output 'my_cmc.pdf' dev-1.txt eval-1.txt
dev-2.txt eval-2.txt
where `my_cmc.pdf` will contain CMC curves for the two experiments.
@@ -270,7 +270,7 @@ each experiment. For example:
.. code-block:: sh
- $bob bio evaluate -v -l 'my_metrics.txt' -o 'my_plots.pdf' {sys1,sys2}/{dev,eval}
+ $bob bio evaluate -e -v -l 'my_metrics.txt' -o 'my_plots.pdf' {sys1,sys2}/{dev,eval}
will output metrics and plots for the two experiments (dev and eval pairs) in
`my_metrics.txt` and `my_plots.pdf`, respectively.