From 293db954ac160d706a479b9aa22e96e4ae3d2ced Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Wed, 25 Apr 2018 08:27:25 +0200
Subject: [PATCH] Change DIC to DIR, add old evaluate.py guoide information
---
bob/bio/base/script/commands.py | 12 ++++++------
bob/bio/base/script/figure.py | 10 +++++-----
bob/bio/base/test/test_commands.py | 6 +++---
doc/experiments.rst | 22 ++++++++++++++++++++--
doc/py_api.rst | 2 +-
setup.py | 2 +-
6 files changed, 36 insertions(+), 18 deletions(-)
diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
index 20cfffa5..8dd82aca 100644
--- a/bob/bio/base/script/commands.py
+++ b/bob/bio/base/script/commands.py
@@ -18,7 +18,7 @@ def rank_option(**kwargs):
return value
return click.option(
'-rk', '--rank', type=click.INT, default=1,
- help='Rank for DIC',
+ help='Provide rank for the command',
callback=callback, show_default=True, **kwargs)(func)
return custom_rank_option
@@ -231,7 +231,7 @@ def cmc(ctx, scores, evaluation, **kargs):
@common_options.figsize_option()
@verbosity_option()
@click.pass_context
-def dic(ctx, scores, evaluation, **kargs):
+def dir(ctx, scores, evaluation, **kargs):
"""Plots the Detection & Identification curve over the FAR
This curve is designed to be used in an open set identification protocol, and
@@ -255,14 +255,14 @@ def dic(ctx, scores, evaluation, **kargs):
:py:func:`bob.bio.base.score.load.five_column` for details.
Examples:
- $ bob bio dic dev-scores
+ $ bob bio dir dev-scores
- $ bob bio dic dev-scores1 eval-scores1 dev-scores2
+ $ bob bio dir dev-scores1 eval-scores1 dev-scores2
eval-scores2
- $ bob bio dic -o my_roc.pdf dev-scores1 eval-scores1
+ $ bob bio dir -o my_roc.pdf dev-scores1 eval-scores1
"""
- process = bio_figure.Dic(ctx, scores, evaluation, load.cmc)
+ process = bio_figure.Dir(ctx, scores, evaluation, load.cmc)
process.run()
@click.command()
diff --git a/bob/bio/base/script/figure.py b/bob/bio/base/script/figure.py
index 46aebc45..660fa7f8 100644
--- a/bob/bio/base/script/figure.py
+++ b/bob/bio/base/script/figure.py
@@ -65,19 +65,19 @@ class Cmc(measure_figure.PlotBase):
)
self._max_R = max(rank, self._max_R)
-class Dic(measure_figure.PlotBase):
- ''' Handles the plotting of DIC'''
+class Dir(measure_figure.PlotBase):
+ ''' Handles the plotting of DIR curve'''
def __init__(self, ctx, scores, evaluation, func_load):
- super(Dic, self).__init__(ctx, scores, evaluation, func_load)
+ super(Dir, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
- self._title = self._title or 'DIC'
+ self._title = self._title or 'DIR curve'
self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR'
def compute(self, idx, input_scores, input_names):
- ''' Plot DIC for dev and eval data using
+ ''' Plot DIR for dev and eval data using
:py:func:`bob.measure.plot.detection_identification_curve`'''
mpl.figure(1)
if self._eval:
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
index c8706072..a685417e 100644
--- a/bob/bio/base/test/test_commands.py
+++ b/bob/bio/base/test/test_commands.py
@@ -224,19 +224,19 @@ def test_cmc():
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
-def test_dic():
+def test_dir():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
runner = CliRunner()
with runner.isolated_filesystem():
- result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
+ result = runner.invoke(commands.dir, ['--no-evaluation', dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0, (result.exit_code, result.output)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
- result = runner.invoke(commands.dic, ['--output', 'test.pdf',
+ result = runner.invoke(commands.dir, ['--output', 'test.pdf',
'--titles', 'A,B',
dev1, test1, dev1, test1])
if result.output:
diff --git a/doc/experiments.rst b/doc/experiments.rst
index 16bd8589..8a0b67e4 100644
--- a/doc/experiments.rst
+++ b/doc/experiments.rst
@@ -227,7 +227,7 @@ file containing the plots. Available plots are:
* ``cmc`` (cumulative match characteristic curve)
-* ``dic`` (detection identification curve)
+* ``dir`` (detection identification rate)
Use the ``--help`` option on the above-cited commands to find-out about more
options.
@@ -243,7 +243,7 @@ For example, to generate a CMC curve from development and evaluation datasets:
where `my_cmc.pdf` will contain CMC curves for the two experiments.
.. note::
- By default, ``det``, ``roc``, ``cmc`` and ``dic`` plot development and
+ By default, ``det``, ``roc``, ``cmc`` and ``dir`` plot development and
evaluation curves on
different plots. You can force gather everything in the same plot using
``--no-split`` option.
@@ -264,6 +264,24 @@ experiment. For example:
will output metrics and plots for the two experiments (dev and eval pairs) in
`my_metrics.txt` and `my_plots.pdf`, respectively.
+Evaluate script (deprecated)
+============================
+
+After the experiment has finished successfully, one or more text file containing all the scores are written.
+To evaluate the experiment, you can use the generic ``evaluate.py`` script, which has properties for all prevalent evaluation types, such as CMC, DIR, ROC and DET plots, as well as computing recognition rates, EER/HTER, Cllr and minDCF.
+Additionally, a combination of different algorithms can be plotted into the same files.
+Just specify all the score files that you want to evaluate using the ``--dev-files`` option, and possible legends for the plots (in the same order) using the ``--legends`` option, and the according plots will be generated.
+For example, to create a ROC curve for the experiment above, use:
+
+.. code-block:: sh
+
+ $ evaluate.py --dev-files results/pca-experiment/male/nonorm/scores-dev --legend MOBIO --roc MOBIO_MALE_ROC.pdf -vv
+
+
+Please note that there exists another file called ``Experiment.info`` inside the result directory.
+This file is a pure text file and contains the complete configuration of the experiment.
+With this configuration it is possible to inspect all default parameters of the algorithms, and even to re-run the exact same experiment.
+
.. _running_in_parallel:
Running in Parallel
diff --git a/doc/py_api.rst b/doc/py_api.rst
index d89f63dc..aa008ee9 100644
--- a/doc/py_api.rst
+++ b/doc/py_api.rst
@@ -113,7 +113,7 @@ Plotting
--------
.. autosummary::
bob.bio.base.script.figure.Cmc
- bob.bio.base.script.figure.Dic
+ bob.bio.base.script.figure.Dir
bob.bio.base.script.figure.Hist
OpenBR conversions
diff --git a/setup.py b/setup.py
index 042c6336..58dbbfcf 100644
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@ setup(
'epc = bob.bio.base.script.commands:epc',
'hist = bob.bio.base.script.commands:hist',
'cmc = bob.bio.base.script.commands:cmc',
- 'dic = bob.bio.base.script.commands:dic',
+ 'dir = bob.bio.base.script.commands:dir',
'gen = bob.bio.base.script.gen:gen',
'evaluate = bob.bio.base.script.commands:evaluate',
],
--
GitLab