Commit 5edb7da6 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

[commands], [test][test_script], [doc][guide] Modificyations related to

default change of --eval option

Was True, is not False, so tests and doc must be modified accordingly
parent fa270494
Pipeline #20851 passed with stage
in 13 minutes and 24 seconds
......@@ -27,16 +27,16 @@ def metrics(ctx, scores, evaluation, **kwargs):
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
evaluation scores are provided, you must use flag `--eval`.
Resulting table format can be changed using the `--tablefmt`.
Examples:
$ bob measure metrics dev-scores
$ bob measure metrics -l results.txt dev-scores1 eval-scores1
$ bob measure metrics -e -l results.txt dev-scores1 eval-scores1
$ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
$ bob measure metrics -e {dev,eval}-scores1 {dev,eval}-scores2
"""
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
......@@ -72,15 +72,15 @@ def roc(ctx, scores, evaluation, **kwargs):
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
evaluation scores are provided, you must use flag `--eval`.
Examples:
$ bob measure roc -v dev-scores
$ bob measure roc -v dev-scores1 eval-scores1 dev-scores2
$ bob measure roc -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure roc -v -o my_roc.pdf dev-scores1 eval-scores1
$ bob measure roc -e -v -o my_roc.pdf dev-scores1 eval-scores1
"""
process = figure.Roc(ctx, scores, evaluation, load.split)
process.run()
......@@ -115,15 +115,15 @@ def det(ctx, scores, evaluation, **kwargs):
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
evaluation scores are provided, you must use flag `--eval`.
Examples:
$ bob measure det -v dev-scores
$ bob measure det -v dev-scores1 eval-scores1 dev-scores2
$ bob measure det -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure det -v -o my_det.pdf dev-scores1 eval-scores1
$ bob measure det -e -v -o my_det.pdf dev-scores1 eval-scores1
"""
process = figure.Det(ctx, scores, evaluation, load.split)
process.run()
......@@ -189,7 +189,7 @@ def hist(ctx, scores, evaluation, **kwargs):
You need to provide one or more development score file(s) for each
experiment. You can also provide evaluation files along with dev files. If
only dev scores are provided, you must use flag `--no-evaluation`.
evaluation scores are provided, you must use flag `--eval`.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
......@@ -198,10 +198,10 @@ def hist(ctx, scores, evaluation, **kwargs):
Examples:
$ bob measure hist -v dev-scores
$ bob measure hist -v dev-scores1 eval-scores1 dev-scores2
$ bob measure hist -e -v dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob measure hist -v --criterion min-hter dev-scores1 eval-scores1
$ bob measure hist -e -v --criterion min-hter dev-scores1 eval-scores1
"""
process = figure.Hist(ctx, scores, evaluation, load.split)
process.run()
......@@ -245,12 +245,12 @@ def evaluate(ctx, scores, evaluation, **kwargs):
Examples:
$ bob measure evaluate -v dev-scores
$ bob measure evaluate -v scores-dev1 scores-eval1 scores-dev2
$ bob measure evaluate -e -v scores-dev1 scores-eval1 scores-dev2
scores-eval2
$ bob measure evaluate -v /path/to/sys-{1,2,3}/scores-{dev,eval}
$ bob measure evaluate -e -v /path/to/sys-{1,2,3}/scores-{dev,eval}
$ bob measure evaluate -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
$ bob measure evaluate -e -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
'''
# first time erase if existing file
ctx.meta['open_mode'] = 'w'
......
......@@ -10,7 +10,7 @@ from .script import commands
def test_metrics():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
result = runner.invoke(commands.metrics, ['--no-evaluation', dev1])
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -22,7 +22,7 @@ def test_metrics():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, [dev1, test1, dev2, test2]
commands.metrics, ['-e', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
......@@ -30,14 +30,14 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-lg',
commands.metrics, ['-e', '-l', 'tmp', dev1, test1, dev2, test2, '-lg',
'A,B']
)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--no-evaluation', dev1, dev2]
commands.metrics, ['-l', 'tmp', dev1, dev2]
)
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -45,8 +45,7 @@ def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--no-evaluation', '--output',
'test.pdf',dev1])
result = runner.invoke(commands.roc, ['--output', 'test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -55,7 +54,7 @@ def test_roc():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--split', '--output',
'test.pdf',
'test.pdf', '-e',
'-ts', 'A,',
dev1, test1, dev2, test2])
if result.output:
......@@ -63,7 +62,7 @@ def test_roc():
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
result = runner.invoke(commands.roc, ['-e', '--output',
'test.pdf', '--legends', 'A,B',
dev1, test1, dev2, test2])
if result.output:
......@@ -75,7 +74,7 @@ def test_det():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--no-evaluation', dev1])
result = runner.invoke(commands.det, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
......@@ -83,7 +82,7 @@ def test_det():
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--split', '--output',
result = runner.invoke(commands.det, ['-e', '--split', '--output',
'test.pdf', '--legends', 'A,B',
'-lc', 'upper-right',
dev1, test1, dev2, test2])
......@@ -93,7 +92,7 @@ def test_det():
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
'test.pdf',
'test.pdf', '-e',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
......@@ -128,14 +127,14 @@ def test_hist():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--no-evaluation', '--criterion',
'min-hter',
result = runner.invoke(commands.hist, ['--criterion',
'min-hter', '--no-line',
'--output', 'HISTO.pdf', '-b',
'30,100', dev1, dev2])
if result.output:
......@@ -143,7 +142,7 @@ def test_hist():
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion', 'eer','--output',
result = runner.invoke(commands.hist, ['-e', '--criterion', 'eer','--output',
'HISTO.pdf', '-b', '30,20',
'-sp', 221, '-lg', 'A,B',
dev1, test1, dev2, test2])
......@@ -159,16 +158,16 @@ def test_evaluate():
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['--no-evaluation', dev1])
result = runner.invoke(commands.evaluate, [dev1])
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, ['--no-evaluation', '--output', 'my_plots.pdf',
commands.evaluate, ['--output', 'my_plots.pdf',
'-n', 300, dev1, dev2])
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, [dev1, test1, dev2, test2])
commands.evaluate, ['-e', dev1, test1, dev2, test2])
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -508,7 +508,7 @@ and FRs are also displayed between parenthesis.
.. note::
Several scores files can be given at once and the metrics will be computed
for each of them separatly. Development and evaluation files must be given by
pairs. When only Development file are provided, ``--no-evaluation`` flag
pairs. When evaluation files are provided, ``--eval`` flag
must be given.
......@@ -535,7 +535,7 @@ on an evaluation set:
.. code-block:: sh
$ bob measure metrics dev-1.txt eval-1.txt
$ bob measure metrics -e dev-1.txt eval-1.txt
[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
==== =================== ===============
.. Development dev-1 Eval. eval-1
......@@ -575,7 +575,7 @@ For example, to generate a DET curve from development and evaluation datasets:
.. code-block:: sh
$bob measure det -v --output "my_det.pdf" -ts "DetDev1,DetEval1,DetDev2,DetEval2"
$bob measure det -e -v --output "my_det.pdf" -ts "DetDev1,DetEval1,DetDev2,DetEval2"
dev-1.txt eval-1.txt dev-2.txt eval-2.txt
where `my_det.pdf` will contain DET plots for the two experiments.
......@@ -600,7 +600,7 @@ experiment. For example:
.. code-block:: sh
$bob measure evaluate -v -l 'my_metrics.txt' -o 'my_plots.pdf' {sys1,sys2}/{dev,eval}
$bob measure evaluate -e -v -l 'my_metrics.txt' -o 'my_plots.pdf' {sys1,sys2}/{dev,eval}
will output metrics and plots for the two experiments (dev and eval pairs) in
`my_metrics.txt` and `my_plots.pdf`, respectively.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment