Merge branch 'theo-cli' into 'theo'

generic plotting script for bob measure

See merge request !52

bob/measure/data/test_m1.txt

+[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
+====  ===================
+..    Development dev-1
+====  ===================
+FMR   6.263% (31/495)
+FNMR  6.208% (28/451)
+FAR   5.924%
+FRR   11.273%
+HTER  8.599%
+====  ===================

bob/measure/data/test_m2.txt

+[Min. criterion: EER] Threshold on Development set `dev-1`: -8.025286e-03
+====  ===================  ===============
+..    Development dev-1    Test test-1
+====  ===================  ===============
+FMR   6.263% (31/495)      5.637% (27/479)
+FNMR  6.208% (28/451)      6.131% (29/473)
+FAR   5.924%               5.366%
+FRR   11.273%              10.637%
+HTER  8.599%               8.001%
+====  ===================  ===============
+[Min. criterion: EER] Threshold on Development set `dev-2`: 1.652567e-02
+====  ===================  ===============
+..    Development dev-2    Test test-2
+====  ===================  ===============
+FMR   4.591% (23/501)      3.333% (16/480)
+FNMR  4.484% (20/446)      7.006% (33/471)
+FAR   4.348%               3.170%
+FRR   9.547%               11.563%
+HTER  6.947%               7.367%
+====  ===================  ===============

bob/measure/load.py

@@ -41,6 +41,34 @@ def split(filename):
                    the first column containing -1 or 1 (i.e. negative or
                    positive) and the second the scores
                      (float).'''.format(filename))
-        return None, None
+        raise
     return (scores[numpy.where(neg_pos == -1)],
             scores[numpy.where(neg_pos == 1)])
+
+def split_files(filenames):
+    """split_files
+
+    Parse a list of files using :py:func:`split`
+
+    Parameters
+    ----------
+
+    filenames :
+        :any:`list`: A list of file paths
+
+    Returns
+    -------
+    :any:`list`: A list of tuples, where each tuple contains the
+    ``negative`` and ``positive`` scores for one probe of the database. Both
+    ``negatives`` and ``positives`` can be either an 1D
+    :py:class:`numpy.ndarray` of type ``float``, or ``None``.
+    """
+    if filenames is None:
+        return None
+    res = []
+    for file_path in filenames:
+        try:
+            res.append(split(file_path))
+        except:
+            raise
+    return res

bob/measure/plot.py

@@ -89,9 +89,9 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
   from . import roc as calc
   out = calc(negatives, positives, npoints)
   if not CAR:
-    return pyplot.plot(100.0 * out[0, :], 100.0 * out[1, :], **kwargs)
+    return pyplot.plot(out[0, :], out[1, :], **kwargs)
   else:
-    return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs)
+    return pyplot.semilogx(out[0, :],(1 - out[1, :]), **kwargs)
 
 
 def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
@@ -142,7 +142,7 @@ def roc_for_far(negatives, positives, far_values=log_values(), **kwargs):
   from matplotlib import pyplot
   from . import roc_for_far as calc
   out = calc(negatives, positives, far_values)
-  return pyplot.semilogx(100.0 * out[0, :], 100.0 * (1 - out[1, :]), **kwargs)
+  return pyplot.semilogx(out[0, :], (1 - out[1, :]), **kwargs)
 
 
 def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
@@ -260,7 +260,7 @@ def epc(dev_negatives, dev_positives, test_negatives, test_positives,
   return pyplot.plot(out[0, :], 100.0 * out[1, :], **kwargs)
 
 
-def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs):
+def det(negatives, positives, npoints=100, **kwargs):
   """Plots Detection Error Trade-off (DET) curve as defined in the paper:
 
   Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
@@ -381,9 +381,9 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs):
   pticks = [ppndf(float(v)) for v in desiredTicks]
   ax = pyplot.gca()  # and finally we set our own tick marks
   ax.set_xticks(pticks)
-  ax.set_xticklabels(desiredLabels, size=axisfontsize)
+  ax.set_xticklabels(desiredLabels)
   ax.set_yticks(pticks)
-  ax.set_yticklabels(desiredLabels, size=axisfontsize)
+  ax.set_yticklabels(desiredLabels)
 
   return retval
 
@@ -481,9 +481,9 @@ def cmc(cmc_scores, logx=True, **kwargs):
   out = calc(cmc_scores)
 
   if logx:
-    pyplot.semilogx(range(1, len(out) + 1), out * 100, **kwargs)
+    pyplot.semilogx(range(1, len(out) + 1), out, **kwargs)
   else:
-    pyplot.plot(range(1, len(out) + 1), out * 100, **kwargs)
+    pyplot.plot(range(1, len(out) + 1), out, **kwargs)
 
   return len(out)
 

bob/measure/script/commands.py

+''' Click commands for ``bob.measure`` '''
+
+
+import click
+from .. import load
+from . import figure
+from . import common_options
+from bob.extension.scripts.click_helper import (verbosity_option,
+                                                open_file_mode_option)
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.eval_option()
+@common_options.table_option()
+@common_options.output_log_metric_option()
+@common_options.criterion_option()
+@common_options.thresholds_option()
+@common_options.far_option()
+@common_options.titles_option()
+@open_file_mode_option()
+@verbosity_option()
+@click.pass_context
+def metrics(ctx, scores, evaluation, **kwargs):
+    """Prints a table that contains FtA, FAR, FRR, FMR, FMNR, HTER for a given
+    threshold criterion (eer or hter).
+
+    You need to provide one or more development score file(s) for each experiment.
+    You can also provide evaluation files along with dev files. If only dev scores
+    are provided, you must use flag `--no-evaluation`.
+
+    Resulting table format can be changed using the `--tablefmt`.
+
+    Examples:
+        $ bob measure metrics dev-scores
+
+        $ bob measure metrics -l results.txt dev-scores1 eval-scores1
+
+        $ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
+    """
+    process = figure.Metrics(ctx, scores, evaluation, load.split_files)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.title_option()
+@common_options.titles_option()
+@common_options.sep_dev_eval_option()
+@common_options.output_plot_file_option(default_out='roc.pdf')
+@common_options.eval_option()
+@common_options.points_curve_option()
+@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
+@common_options.x_rotation_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
+@common_options.lines_at_option()
+@common_options.const_layout_option()
+@common_options.figsize_option()
+@common_options.style_option()
+@verbosity_option()
+@click.pass_context
+def roc(ctx, scores, evaluation, **kwargs):
+    """Plot ROC (receiver operating characteristic) curve:
+    The plot will represent the false match rate on the horizontal axis and the
+    false non match rate on the vertical axis.  The values for the axis will be
+    computed using :py:func:`bob.measure.roc`.
+
+    You need to provide one or more development score file(s) for each experiment.
+    You can also provide evaluation files along with dev files. If only dev scores
+    are provided, you must use flag `--no-evaluation`.
+
+    Examples:
+        $ bob measure roc dev-scores
+
+        $ bob measure roc dev-scores1 eval-scores1 dev-scores2
+        eval-scores2
+
+        $ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
+    """
+    process = figure.Roc(ctx, scores, evaluation, load.split_files)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='det.pdf')
+@common_options.title_option()
+@common_options.titles_option()
+@common_options.sep_dev_eval_option()
+@common_options.eval_option()
+@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
+@common_options.x_rotation_option(dflt=45)
+@common_options.x_label_option()
+@common_options.y_label_option()
+@common_options.points_curve_option()
+@common_options.lines_at_option()
+@common_options.const_layout_option()
+@common_options.figsize_option()
+@common_options.style_option()
+@verbosity_option()
+@click.pass_context
+def det(ctx, scores, evaluation, **kwargs):
+    """Plot DET (detection error trade-off) curve:
+    modified ROC curve which plots error rates on both axes
+    (false positives on the x-axis and false negatives on the y-axis)
+
+    You need to provide one or more development score file(s) for each experiment.
+    You can also provide evaluation files along with dev files. If only dev scores
+    are provided, you must use flag `--no-evaluation`.
+
+    Examples:
+        $ bob measure det dev-scores
+
+        $ bob measure det dev-scores1 eval-scores1 dev-scores2
+        eval-scores2
+
+        $ bob measure det -o my_det.pdf dev-scores1 eval-scores1
+    """
+    process = figure.Det(ctx, scores, evaluation, load.split_files)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(eval_mandatory=True, nargs=-1)
+@common_options.output_plot_file_option(default_out='epc.pdf')
+@common_options.title_option()
+@common_options.titles_option()
+@common_options.points_curve_option()
+@common_options.const_layout_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
+@common_options.figsize_option()
+@common_options.style_option()
+@verbosity_option()
+@click.pass_context
+def epc(ctx, scores, **kwargs):
+    """Plot EPC (expected performance curve):
+    plots the error rate on the eval set depending on a threshold selected
+    a-priori on the development set and accounts for varying relative cost
+    in [0; 1] of FPR and FNR when calculating the threshold.
+
+    You need to provide one or more development score and eval file(s)
+    for each experiment.
+
+    Examples:
+        $ bob measure epc dev-scores eval-scores
+
+        $ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
+    """
+    process = figure.Epc(ctx, scores, True, load.split_files)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.output_plot_file_option(default_out='hist.pdf')
+@common_options.eval_option()
+@common_options.n_bins_option()
+@common_options.criterion_option()
+@common_options.thresholds_option()
+@common_options.const_layout_option()
+@common_options.show_dev_option()
+@common_options.print_filenames_option()
+@common_options.title_option()
+@common_options.titles_option()
+@common_options.figsize_option()
+@common_options.style_option()
+@verbosity_option()
+@click.pass_context
+def hist(ctx, scores, evaluation, **kwargs):
+    """ Plots histograms of positive and negatives along with threshold
+    criterion.
+
+    You need to provide one or more development score file(s) for each experiment.
+    You can also provide evaluation files along with dev files. If only dev scores
+    are provided, you must use flag `--no-evaluation`.
+
+    By default, when eval-scores are given, only eval-scores histograms are
+    displayed with threshold line
+    computed from dev-scores. If you want to display dev-scores distributions
+    as well, use ``--show-dev`` option.
+
+    Examples:
+        $ bob measure hist dev-scores
+
+        $ bob measure hist dev-scores1 eval-scores1 dev-scores2
+        eval-scores2
+
+        $ bob measure hist --criter hter --show-dev dev-scores1 eval-scores1
+    """
+    process = figure.Hist(ctx, scores, evaluation, load.split_files)
+    process.run()
+
+@click.command()
+@common_options.scores_argument(nargs=-1)
+@common_options.titles_option()
+@common_options.sep_dev_eval_option()
+@common_options.table_option()
+@common_options.eval_option()
+@common_options.output_log_metric_option()
+@common_options.output_plot_file_option(default_out='eval_plots.pdf')
+@common_options.points_curve_option()
+@common_options.n_bins_option()
+@common_options.lines_at_option()
+@common_options.const_layout_option()
+@common_options.figsize_option()
+@common_options.style_option()
+@verbosity_option()
+@click.pass_context
+def evaluate(ctx, scores, evaluation, **kwargs):
+    '''Runs error analysis on score sets
+
+    \b
+    1. Computes the threshold using either EER or min. HTER criteria on
+       development set scores
+    2. Applies the above threshold on evaluation set scores to compute the HTER, if a
+       eval-score set is provided
+    3. Reports error rates on the console
+    4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
+       file
+
+
+    You need to provide 2 score files for each biometric system in this order:
+
+    \b
+    * development scores
+    * evaluation scores
+
+    Examples:
+        $ bob measure evaluate dev-scores
+
+        $ bob measure evaluate scores-dev1 scores-eval1 scores-dev2
+        scores-eval2
+
+        $ bob measure evaluate /path/to/sys-{1,2,3}/scores-{dev,eval}
+
+        $ bob measure evaluate -l metrics.txt -o my_plots.pdf dev-scores eval-scores
+    '''
+    # first time erase if existing file
+    ctx.meta['open_mode'] = 'w'
+    click.echo("Computing metrics with EER...")
+    ctx.meta['criter'] = 'eer'  # no criterion passed to evaluate
+    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
+    # second time, appends the content
+    ctx.meta['open_mode'] = 'a'
+    click.echo("Computing metrics with HTER...")
+    ctx.meta['criter'] = 'hter'  # no criterion passed in evaluate
+    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
+    if 'log' in ctx.meta:
+        click.echo("[metrics] => %s" % ctx.meta['log'])
+
+    # avoid closing pdf file before all figures are plotted
+    ctx.meta['closef'] = False
+    if evaluation:
+        click.echo("Starting evaluate with dev and eval scores...")
+    else:
+        click.echo("Starting evaluate with dev scores only...")
+    click.echo("Computing ROC...")
+    # set axes limits for ROC
+    ctx.forward(roc) # use class defaults plot settings
+    click.echo("Computing DET...")
+    ctx.forward(det) # use class defaults plot settings
+    if evaluation:
+        click.echo("Computing EPC...")
+        ctx.forward(epc) # use class defaults plot settings
+    # the last one closes the file
+    ctx.meta['closef'] = True
+    click.echo("Computing score histograms...")
+    ctx.meta['criter'] = 'eer'  # no criterion passed in evaluate
+    ctx.forward(hist)
+    click.echo("Evaluate successfully completed!")
+    click.echo("[plots] => %s" % (ctx.meta['output']))

bob/measure/script/common_options.py

+'''Stores click common options for plots'''
+
+import logging
+import click
+from click.types import INT, FLOAT
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_pdf import PdfPages
+from bob.extension.scripts.click_helper import (bool_option, list_float_option)
+
+LOGGER = logging.getLogger(__name__)
+
+def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
+    """Get the argument for scores, and add `dev-scores` and `eval-scores` in
+    the context when `--evaluation` flag is on (default)
+
+    Parameters
+    ----------
+    eval_mandatory :
+        If evaluation files are mandatory
+    min_len :
+        The min lenght of inputs files that are needed. If eval_mandatory is
+        True, this quantity is multiplied by 2.
+
+    Returns
+    -------
+     callable
+      A decorator to be used for adding score arguments for click commands
+    """
+    def custom_scores_argument(func):
+        def callback(ctx, param, value):
+            length = len(value)
+            min_arg = min_len or 1
+            ctx.meta['min_arg'] = min_arg
+            if length < min_arg:
+                raise click.BadParameter(
+                    'You must provide at least %d score files' % min_arg,
+                    ctx=ctx
+                )
+            else:
+                ctx.meta['scores'] = value
+                step = 1
+                if eval_mandatory or ctx.meta['evaluation']:
+                    step = 2
+                    if (length % (min_arg * 2)) != 0:
+                        pref = 'T' if eval_mandatory else \
+                                ('When `--evaluation` flag is on t')
+                        raise click.BadParameter(
+                            '%sest-score(s) must '
+                            'be provided along with dev-score(s). '
+                            'You must provide at least %d score files.' \
+                            % (pref, min_arg * 2), ctx=ctx
+                        )
+                for arg in range(min_arg):
+                    ctx.meta['dev_scores_%d' % arg] = [
+                        value[i] for i in range(arg * step, length,
+                                                min_arg * step)
+                    ]
+                    if step > 1:
+                        ctx.meta['eval_scores_%d' % arg] = [
+                            value[i] for i in range((arg * step + 1),
+                                                    length, min_arg * step)
+                        ]
+                ctx.meta['n_sys'] = len(ctx.meta['dev_scores_0'])
+                if 'titles' in ctx.meta and \
+                   len(ctx.meta['titles']) != ctx.meta['n_sys']:
+                    raise click.BadParameter(
+                        '#titles not equal to #sytems', ctx=ctx
+                    )
+            return value
+        return click.argument(
+            'scores', type=click.Path(exists=True),
+            callback=callback, **kwargs
+        )(func)
+    return custom_scores_argument
+
+def eval_option(**kwargs):
+    '''Get option flag to say if eval-scores are provided'''
+    return bool_option(
+        'evaluation', 'e', 'If set, evaluation scores must be provided',
+        dflt=True
+    )
+
+def sep_dev_eval_option(dflt=True, **kwargs):
+    '''Get option flag to say if dev and eval plots should be in different
+    plots'''
+    return bool_option(
+        'split', 's','If set, evaluation and dev curve in different plots',
+        dflt
+    )
+
+def cmc_option(**kwargs):
+    '''Get option flag to say if cmc scores'''
+    return bool_option('cmc', 'C', 'If set, CMC score files are provided')
+
+def semilogx_option(dflt=False, **kwargs):
+    '''Option to use semilog X-axis'''
+    return bool_option('semilogx', 'G', 'If set, use semilog on X axis', dflt)
+
+def show_dev_option(dflt=False, **kwargs):
+    '''Option to tell if should show dev histo'''
+    return bool_option('show-dev', 'D', 'If set, show dev histograms', dflt)
+
+def print_filenames_option(dflt=True, **kwargs):
+    '''Option to tell if filenames should be in the title'''
+    return bool_option('show-fn', 'P', 'If set, show filenames in title', dflt)
+
+def const_layout_option(dflt=True, **kwargs):
+    '''Option to set matplotlib constrained_layout'''
+    def custom_layout_option(func):
+        def callback(ctx, param, value):
+            ctx.meta['clayout'] = value
+            plt.rcParams['figure.constrained_layout.use'] = value
+            return value
+        return click.option(
+            '-Y', '--clayout/--no-clayout', default=dflt, show_default=True,
+            help='(De)Activate constrained layout',
+            callback=callback, **kwargs)(func)
+    return custom_layout_option
+
+def axes_val_option(dflt=None, **kwargs):
+    ''' Option for setting min/max values on axes '''
+    return list_float_option(
+        name='axlim', short_name='L',
+        desc='min/max axes values separated by commas (e.g. ``--axlim '
+        ' 0.1,100,0.1,100``)',
+        nitems=4, dflt=dflt, **kwargs
+    )
+
+def thresholds_option(**kwargs):
+    ''' Option to give a list of thresholds '''
+    return list_float_option(
+        name='thres', short_name='T',
+        desc='Given threshold for metrics computations, e.g. '
+        '0.005,0.001,0.056',
+        nitems=None, dflt=None, **kwargs
+    )
+
+def lines_at_option(**kwargs):
+    '''Get option to draw const far line'''
+    return list_float_option(
+        name='lines-at', short_name='la',
+        desc='If given, draw veritcal lines at the given axis positions',
+        nitems=None, dflt=None, **kwargs
+    )
+
+def x_rotation_option(dflt=0, **kwargs):
+    '''Get option for rotartion of the x axis lables'''
+    def custom_x_rotation_option(func):
+        def callback(ctx, param, value):
+            value = abs(value)
+            ctx.meta['x_rotation'] = value
+            return value
+        return click.option(
+            '-r', '--x-rotation', type=click.INT, default=dflt, show_default=True,
+            help='X axis labels ration',
+            callback=callback, **kwargs)(func)
+    return custom_x_rotation_option
+
+def cost_option(**kwargs):
+    '''Get option to get cost for FAR'''
+    def custom_cost_option(func):
+        def callback(ctx, param, value):
+            if value < 0 or value > 1:
+                raise click.BadParameter("Cost for FAR must be betwen 0 and 1")
+            ctx.meta['cost'] = value
+            return value
+        return click.option(
+            '-C', '--cost', type=float, default=0.99, show_default=True,
+            help='Cost for FAR in minDCF',
+            callback=callback, **kwargs)(func)
+    return custom_cost_option
+
+def points_curve_option(**kwargs):
+    '''Get the number of points use to draw curves'''
+    def custom_points_curve_option(func):
+        def callback(ctx, param, value):
+            if value < 2:
+                raise click.BadParameter(
+                    'Number of points to draw curves must be greater than 1'
+                    , ctx=ctx
+                )
+            ctx.meta['points'] = value
+            return value
+        return click.option(
+            '-n', '--points', type=INT, default=100, show_default=True,
+            help='The number of points use to draw curves in plots',
+            callback=callback, **kwargs)(func)
+    return custom_points_curve_option
+
+def n_bins_option(**kwargs):
+    '''Get the number of bins in the histograms'''
+    def custom_n_bins_option(func):
+        def callback(ctx, param, value):
+            if value is None:
+                value = 'auto'
+            elif value < 2:
+                raise click.BadParameter(
+                    'Number of bins must be greater than 1'
+                    , ctx=ctx
+                )
+            ctx.meta['n_bins'] = value
+            return value
+        return click.option(
+            '-b', '--nbins', type=INT, default=None,