Add options for axis labels, modifications accoding to the move of click...

Add options for axis labels, modifications accoding to the move of click options to bob.extension, print assert outputs for tests

bob/bio/base/script/commands.py

@@ -6,7 +6,8 @@ import bob.bio.base.script.figure as bio_figure
 import bob.measure.script.figure as measure_figure
 from ..score import load
 from bob.measure.script import common_options
-from bob.extension.scripts.click_helper import verbosity_option
+from bob.extension.scripts.click_helper import (verbosity_option,
+                                                open_file_mode_option)
 
 FUNC_SPLIT = lambda x: load.load_files(x, load.split)
 FUNC_CMC = lambda x: load.load_files(x, load.cmc)
@@ -15,13 +16,13 @@ FUNC_CMC = lambda x: load.load_files(x, load.cmc)
 @common_options.scores_argument(nargs=-1)
 @common_options.table_option()
 @common_options.eval_option()
-@common_options.open_file_mode_option()
 @common_options.output_plot_metric_option()
 @common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
 @common_options.cost_option()
 @common_options.thresholds_option()
 @common_options.far_option()
 @common_options.titles_option()
+@open_file_mode_option()
 @verbosity_option()
 @click.pass_context
 def metrics(ctx, scores, evaluation, **kargs):
@@ -65,7 +66,9 @@ def metrics(ctx, scores, evaluation, **kargs):
 @common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
 @common_options.axis_fontsize_option()
 @common_options.x_rotation_option()
-@common_options.fmr_line_at_option()
+@common_options.lines_at_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
 @common_options.const_layout_option()
 @verbosity_option()
 @click.pass_context
@@ -90,13 +93,15 @@ def roc(ctx, scores, evaluation, **kargs):
 
         $ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1
     """
-    process = measure_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
+    process = bio_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
     process.run()
 
 @click.command()
 @common_options.scores_argument(nargs=-1)
 @common_options.output_plot_file_option(default_out='det.pdf')
 @common_options.titles_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
 @common_options.sep_dev_eval_option()
 @common_options.eval_option()
 @common_options.axis_fontsize_option(dflt=6)
@@ -126,7 +131,7 @@ def det(ctx, scores, evaluation, **kargs):
 
         $ bob bio det -o my_det.pdf dev-scores1 eval-scores1
     """
-    process = measure_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
+    process = bio_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
     process.run()
 
 @click.command()
@@ -294,7 +299,7 @@ def hist(ctx, scores, evaluation, **kwargs):
 @common_options.output_plot_metric_option()
 @common_options.output_plot_file_option(default_out='eval_plots.pdf')
 @common_options.points_curve_option()
-@common_options.fmr_line_at_option()
+@common_options.lines_at_option()
 @common_options.cost_option()
 @common_options.rank_option()
 @common_options.far_option()

bob/bio/base/script/figure.py

@@ -7,6 +7,22 @@ import bob.measure
 from bob.measure import plot
 from tabulate import tabulate
 
+class Roc(measure_figure.Roc):
+    def __init__(self, ctx, scores, evaluation, func_load):
+        super(Roc, self).__init__(ctx, scores, evaluation, func_load)
+        self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta  or \
+        ctx.meta['x_label'] is None else ctx.meta['x_label']
+        self._y_label = '1 - False Non Match Rate' if 'y_label' not in \
+        ctx.meta or ctx.meta['y_label'] is None else ctx.meta['y_label']
+
+class Det(measure_figure.Det):
+    def __init__(self, ctx, scores, evaluation, func_load):
+        super(Det, self).__init__(ctx, scores, evaluation, func_load)
+        self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
+        ctx.meta['x_label'] is None else ctx.meta['x_label']
+        self._y_label = 'False Non Match Rate' if 'y_label' not in ctx.meta or\
+        ctx.meta['y_label'] is None else ctx.meta['y_label']
+
 class Cmc(measure_figure.PlotBase):
     ''' Handles the plotting of Cmc
 
@@ -20,9 +36,9 @@ class Cmc(measure_figure.PlotBase):
         super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
         self._semilogx = True if 'semilogx' not in ctx.meta else\
         ctx.meta['semilogx']
-        self._title = 'CMC'
-        self._x_label = 'Rank'
-        self._y_label = 'Probability'
+        self._title = self._title or 'CMC'
+        self._x_label = self._x_label or 'Rank'
+        self._y_label = self._y_label or 'Identification rate'
         self._max_R = 0
 
     def compute(self, idx, dev_score, dev_file=None,
@@ -79,9 +95,9 @@ class Dic(measure_figure.PlotBase):
         self._semilogx = True if 'semilogx' not in ctx.meta else\
                 ctx.meta['semilogx']
         self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
-        self._title = 'DIC'
-        self._x_label = 'FAR'
-        self._y_label = 'DIR'
+        self._title = self._title or 'DIC'
+        self._x_label = self._title or 'FAR'
+        self._y_label = self._title or 'DIR'
 
     def compute(self, idx, dev_score, dev_file=None,
                 eval_score=None, eval_file=None):

bob/bio/base/script/gen.py

+"""Generate random scores.
+"""
+import pkg_resources  # to make sure bob gets imported properly
+import os
+import logging
+import numpy
+import random as rd
+import click
+from click.types import FLOAT
+from bob.extension.scripts.click_helper import verbosity_option
+from bob.core import random
+from bob.io.base import create_directories_safe
+
+logger = logging.getLogger(__name__)
+
+NUM_NEG = 5000
+NUM_POS = 5000
+
+def gen_score_distr(mean_neg, mean_pos, sigma_neg=10, sigma_pos=10):
+    """Generate scores from normal distributions
+
+    Parameters
+    ----------
+
+    mean_neg : float
+        Mean for negative scores
+    mean_pos : float
+        Mean for positive scores
+    sigma_neg : float
+        STDev for negative scores
+    sigma_pos : float
+        STDev for positive scores
+
+    Returns
+    -------
+
+    neg_scores : array_like
+        Negatives scores
+    pos_scores : array_like
+        Positive scores
+    """
+    mt = random.mt19937()  # initialise the random number generator
+
+    neg_generator = random.normal(numpy.float32, mean_neg, sigma_neg)
+    pos_generator = random.normal(numpy.float32, mean_pos, sigma_pos)
+
+    neg_scores = [neg_generator(mt) for _ in range(NUM_NEG)]
+    pos_scores = [pos_generator(mt) for _ in range(NUM_NEG)]
+
+    return neg_scores, pos_scores
+
+def write_scores_to_file(pos, neg, filename, n_sys=1, five_col=False):
+    """ Writes score distributions
+
+    Parameters
+    ----------
+
+    pos : array_like
+        Scores for positive samples.
+    neg : array_like
+        Scores for negative samples.
+    filename : str
+        The path to write the score to.
+    n_sys : int
+        Number of different systems
+    five_col : bool
+        If 5-colum format, else 4-column
+    """
+    create_directories_safe(os.path.dirname(filename))
+    s_names = ['s%d' % i for i in range(n_sys)]
+    with open(filename, 'wt') as f:
+        for i in pos:
+            s_name = rd.choice(s_names)
+            s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
+            f.write('x%sx %s %f\n' % (s_five, s_name, i))
+        for i in neg:
+            s_name = rd.choice(s_names)
+            s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
+            f.write('x%sy %s %f\n' % (s_five, s_name, i))
+
+@click.command()
+@click.argument('outdir')
+@click.option('-mm', '--mean-match', default=10, type=FLOAT, show_default=True)
+@click.option('-mnm', '--mean-non-match', default=-10, type=FLOAT, show_default=True)
+@click.option('-n', '--n-sys', default=1, type=click.INT, show_default=True)
+@click.option('--five-col/--four-col', default=False, show_default=True)
+@verbosity_option()
+def gen(outdir, mean_match, mean_non_match, n_sys, five_col):
+    """Generate random scores.
+    Generates random scores in 4col or 5col format. The scores are generated 
+    using Gaussian distribution whose mean is an input
+    parameter. The generated scores can be used as hypothetical datasets.
+    """
+    # Generate the data
+    neg_dev, pos_dev = gen_score_distr(mean_non_match, mean_match)
+    neg_eval, pos_eval = gen_score_distr(mean_non_match, mean_match)
+
+    # Write the data into files
+    write_scores_to_file(neg_dev, pos_dev,
+                         os.path.join(outdir, 'scores-dev'), n_sys, five_col)
+    write_scores_to_file(neg_eval, pos_eval,
+                         os.path.join(outdir, 'scores-eval'), n_sys, five_col)

bob/bio/base/test/test_commands.py

@@ -15,7 +15,7 @@ def test_metrics():
     with runner.isolated_filesystem():
         with open('tmp', 'w') as f:
             f.write(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     dev2 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/dev-5col.txt')
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -28,19 +28,19 @@ def test_metrics():
         )
         with open('tmp', 'w') as f:
             f.write(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
-            commands.metrics, ['-l', 'tmp', '-t', 'A,B',
+            commands.metrics, ['-l', 'tmp', '-ts', 'A,B',
                                dev1, test1, dev2, test2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     with runner.isolated_filesystem():
         result = runner.invoke(
             commands.metrics, ['-l', 'tmp', dev1, test2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
@@ -48,7 +48,7 @@ def test_metrics():
                                '--criter', 'mindcf', '--cost', 0.9,
                                dev1, test2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
@@ -56,34 +56,34 @@ def test_metrics():
                                '--criter', 'mindcf', '--cost', 0.9,
                                dev1]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
             commands.metrics, ['--criter', 'cllr', dev1, test2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
             commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'cllr',
                                '--cost', 0.9, dev1]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
             commands.metrics, ['--criter', 'rr', '-T',
                                '0.1', dev1, test2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(
             commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'rr',
                                dev1, dev2]
         )
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 
 def test_roc():
@@ -95,7 +95,7 @@ def test_roc():
                                               'test.pdf',dev1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     dev2 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/dev-5col.txt')
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -108,7 +108,7 @@ def test_roc():
                                               dev1, test1, dev2, test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(commands.roc, ['--output',
@@ -116,7 +116,7 @@ def test_roc():
                                               dev1, test1, dev2, test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 
 def test_det():
@@ -127,7 +127,7 @@ def test_det():
         result = runner.invoke(commands.det, ['--no-evaluation', dev1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     dev2 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/dev-5col.txt')
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -140,14 +140,14 @@ def test_det():
                                               dev1, test1, dev2, test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     with runner.isolated_filesystem():
         result = runner.invoke(commands.det, ['--output',
                                               'test.pdf',
                                               dev1, test1, dev2, test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 def test_epc():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -159,7 +159,7 @@ def test_epc():
         result = runner.invoke(commands.epc, [dev1, test1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     dev2 = pkg_resources.resource_filename('bob.bio.base.test',
                                            'data/dev-4col.tar.gz')
     test2 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -170,7 +170,7 @@ def test_epc():
                                               dev1, test1, dev2, test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 def test_hist():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -186,7 +186,7 @@ def test_hist():
         result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
@@ -194,16 +194,16 @@ def test_hist():
                                                30,'--no-evaluation', dev1, dev2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
     with runner.isolated_filesystem():
         result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
                                                'HISTO.pdf', '-b', 30, '-F', 3,
-                                               '-t', 'A,B', dev1, test1, dev2,
+                                               '-ts', 'A,B', dev1, test1, dev2,
                                                test2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 def test_cmc():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -213,7 +213,7 @@ def test_cmc():
         result = runner.invoke(commands.cmc, ['--no-evaluation', dev1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
                                             'data/scores-cmc-4col.txt')
     with runner.isolated_filesystem():
@@ -222,7 +222,7 @@ def test_cmc():
                                               dev1, test1, dev1, test1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 def test_dic():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -232,7 +232,7 @@ def test_dic():
         result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
     test1 = pkg_resources.resource_filename('bob.bio.base.test',
                                             'data/scores-nonorm-openset-dev')
     with runner.isolated_filesystem():
@@ -241,7 +241,7 @@ def test_dic():
                                               dev1, test1, dev1, test1])
         if result.output:
             click.echo(result.output)
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
 def test_evaluate():
     dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -257,15 +257,15 @@ def test_evaluate():
     with runner.isolated_filesystem():
         result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
                                                    '--no-evaluation', dev1, dev2])
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
         result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
                                                    dev1, dev2])
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 
         result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
                                                    dev1, test1, dev2, test2])
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
         result = runner.invoke(commands.evaluate, ['-f', 0.01,
                                                    dev1, test1, dev2, test2])
-        assert result.exit_code == 0
+        assert result.exit_code == 0, (result.exit_code, result.output)
 

setup.py

@@ -145,6 +145,7 @@ setup(
         'hist              = bob.bio.base.script.commands:hist',
         'cmc               = bob.bio.base.script.commands:cmc',
         'dic               = bob.bio.base.script.commands:dic',
+        'gen               = bob.bio.base.script.gen:gen',
         'evaluate          = bob.bio.base.script.commands:evaluate',
       ],