Change DIC to DIR, add old evaluate.py guoide information

bob/bio/base/script/commands.py

@@ -18,7 +18,7 @@ def rank_option(**kwargs):
             return value
         return click.option(
             '-rk', '--rank', type=click.INT, default=1,
-            help='Rank for DIC',
+            help='Provide rank for the command',
             callback=callback, show_default=True, **kwargs)(func)
     return custom_rank_option
 
@@ -231,7 +231,7 @@ def cmc(ctx, scores, evaluation, **kargs):
 @common_options.figsize_option()
 @verbosity_option()
 @click.pass_context
-def dic(ctx, scores, evaluation, **kargs):
+def dir(ctx, scores, evaluation, **kargs):
     """Plots the Detection & Identification curve over the FAR
 
     This curve is designed to be used in an open set identification protocol, and
@@ -255,14 +255,14 @@ def dic(ctx, scores, evaluation, **kargs):
     :py:func:`bob.bio.base.score.load.five_column` for details.
 
     Examples:
-        $ bob bio dic dev-scores
+        $ bob bio dir dev-scores
 
-        $ bob bio dic dev-scores1 eval-scores1 dev-scores2
+        $ bob bio dir dev-scores1 eval-scores1 dev-scores2
         eval-scores2
 
-        $ bob bio dic -o my_roc.pdf dev-scores1 eval-scores1
+        $ bob bio dir -o my_roc.pdf dev-scores1 eval-scores1
     """
-    process = bio_figure.Dic(ctx, scores, evaluation, load.cmc)
+    process = bio_figure.Dir(ctx, scores, evaluation, load.cmc)
     process.run()
 
 @click.command()

bob/bio/base/script/figure.py

@@ -65,19 +65,19 @@ class Cmc(measure_figure.PlotBase):
             )
             self._max_R = max(rank, self._max_R)
 
-class Dic(measure_figure.PlotBase):
-    ''' Handles the plotting of DIC'''
+class Dir(measure_figure.PlotBase):
+    ''' Handles the plotting of DIR curve'''
     def __init__(self, ctx, scores, evaluation, func_load):
-        super(Dic, self).__init__(ctx, scores, evaluation, func_load)
+        super(Dir, self).__init__(ctx, scores, evaluation, func_load)
         self._semilogx = True if 'semilogx' not in ctx.meta else\
                 ctx.meta['semilogx']
         self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
-        self._title = self._title or 'DIC'
+        self._title = self._title or 'DIR curve'
         self._x_label = self._title or 'FAR'
         self._y_label = self._title or 'DIR'
 
     def compute(self, idx, input_scores, input_names):
-        ''' Plot DIC for dev and eval data using
+        ''' Plot DIR for dev and eval data using
         :py:func:`bob.measure.plot.detection_identification_curve`'''
         mpl.figure(1)
         if self._eval:

bob/bio/base/test/test_commands.py

@@ -224,19 +224,19 @@ def test_cmc():
             click.echo(result.output)
         assert result.exit_code == 0, (result.exit_code, result.output)
 
-def test_dic():
+def test_dir():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/scores-nonorm-openset-dev')
     runner = CliRunner()
     with runner.isolated_filesystem():
-        result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
+        result = runner.invoke(commands.dir, ['--no-evaluation', dev1, '--rank', 2])
         if result.output:
             click.echo(result.output)
         assert result.exit_code == 0, (result.exit_code, result.output)
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
                                             'data/scores-nonorm-openset-dev')
     with runner.isolated_filesystem():
-        result = runner.invoke(commands.dic, ['--output', 'test.pdf',
+        result = runner.invoke(commands.dir, ['--output', 'test.pdf',
                                               '--titles', 'A,B',
                                               dev1, test1, dev1, test1])
         if result.output:

doc/experiments.rst

@@ -227,7 +227,7 @@ file containing the plots. Available plots are:
 
 *  ``cmc`` (cumulative match characteristic curve)
 
-*  ``dic`` (detection identification curve)
+*  ``dir`` (detection identification rate)
 
 Use the ``--help`` option on the above-cited commands to find-out about more
 options.
@@ -243,7 +243,7 @@ For example, to generate a CMC curve from development and evaluation datasets:
 where `my_cmc.pdf` will contain CMC curves for the two experiments.
 
 .. note::
-    By default, ``det``, ``roc``, ``cmc`` and ``dic`` plot development and
+    By default, ``det``, ``roc``, ``cmc`` and ``dir`` plot development and
     evaluation curves on
     different plots. You can force gather everything in the same plot using
     ``--no-split`` option.
@@ -264,6 +264,24 @@ experiment. For example:
 will output metrics and plots for the two experiments (dev and eval pairs) in
 `my_metrics.txt` and `my_plots.pdf`, respectively.
 
+Evaluate script (deprecated)
+============================
+
+After the experiment has finished successfully, one or more text file containing all the scores are written.
+To evaluate the experiment, you can use the generic ``evaluate.py`` script, which has properties for all prevalent evaluation types, such as CMC, DIR, ROC and DET plots, as well as computing recognition rates, EER/HTER, Cllr and minDCF.
+Additionally, a combination of different algorithms can be plotted into the same files.
+Just specify all the score files that you want to evaluate using the ``--dev-files`` option, and possible legends for the plots (in the same order) using the ``--legends`` option, and the according plots will be generated.
+For example, to create a ROC curve for the experiment above, use:
+
+.. code-block:: sh
+
+   $ evaluate.py --dev-files results/pca-experiment/male/nonorm/scores-dev --legend MOBIO --roc MOBIO_MALE_ROC.pdf -vv
+
+
+Please note that there exists another file called ``Experiment.info`` inside the result directory.
+This file is a pure text file and contains the complete configuration of the experiment.
+With this configuration it is possible to inspect all default parameters of the algorithms, and even to re-run the exact same experiment.
+
 .. _running_in_parallel:
 
 Running in Parallel

doc/py_api.rst

@@ -113,7 +113,7 @@ Plotting
 --------
 .. autosummary::
    bob.bio.base.script.figure.Cmc
-   bob.bio.base.script.figure.Dic
+   bob.bio.base.script.figure.Dir
    bob.bio.base.script.figure.Hist
 
 OpenBR conversions

setup.py

@@ -144,7 +144,7 @@ setup(
         'epc               = bob.bio.base.script.commands:epc',
         'hist              = bob.bio.base.script.commands:hist',
         'cmc               = bob.bio.base.script.commands:cmc',
-        'dic               = bob.bio.base.script.commands:dic',
+        'dir               = bob.bio.base.script.commands:dir',
         'gen               = bob.bio.base.script.gen:gen',
         'evaluate          = bob.bio.base.script.commands:evaluate',
       ],