commands.py 8.08 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
''' Click commands for ``bob.measure`` '''


import click
from .. import load
from . import figure
from . import common_options
from bob.extension.scripts.click_helper import verbosity_option

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.test_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, test, **kargs):
    """Prints a single output line that contains all info for a given
    criterion (eer or hter).

    You need provide one or more development score file(s) for each experiment.
    You can also provide test files along with dev files but the flag `--test`
    is required in that case.

    Resulting table format can be changer using the `--tablefmt`. Default
    formats are `fancy_grid` when output in the terminal and `latex` when
    written in a log file (see `--log`)

    Examples:
        $ bob measure metrics dev-scores

        $ bob measure metrics --test -l results.txt dev-scores1 test-scores1

        $ bob measure metrics --test {dev,test}-scores1 {dev,test}-scores2
    """
    process = figure.Metrics(ctx, scores, test, load.split_files)
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.test_option()
@common_options.points_curve_option()
@common_options.semilogx_option(True)
@common_options.min_x_axis_val_option()
@common_options.max_x_axis_val_option(dflt=1)
@common_options.min_y_axis_val_option(dflt=0)
@common_options.max_y_axis_val_option(dflt=1)
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@verbosity_option()
@click.pass_context
def roc(ctx, scores, test, **kargs):
    """Plot ROC (receiver operating characteristic) curve:
    The plot will represent the false match rate on the horizontal axis and the
    false non match rate on the vertical axis.  The values for the axis will be
    computed using :py:func:`bob.measure.roc`.

    You need provide one or more development score file(s) for each experiment.
    You can also provide test files along with dev files but the flag `--test`
    is required in that case.

    Examples:
        $ bob measure roc dev-scores

        $ bob measure roc --test dev-scores1 test-scores1 dev-scores2
        test-scores2

        $ bob measure roc --test -o my_roc.pdf dev-scores1 test-scores1
    """
    process = figure.Roc(ctx, scores, test, load.split_files)
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.test_option()
@common_options.min_x_axis_val_option(dflt=0.01)
@common_options.max_x_axis_val_option(dflt=95)
@common_options.min_y_axis_val_option(dflt=0.01)
@common_options.max_y_axis_val_option(dflt=95)
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
@common_options.points_curve_option()
@verbosity_option()
@click.pass_context
def det(ctx, scores, test, **kargs):
    """Plot DET (detection error trade-off) curve:
    modified ROC curve which plots error rates on both axes
    (false positives on the x-axis and false negatives on the y-axis)

    You need provide one or more development score file(s) for each experiment.
    You can also provide test files along with dev files but the flag `--test`
    is required in that case.

    Examples:
        $ bob measure det dev-scores

        $ bob measure det --test dev-scores1 test-scores1 dev-scores2
        test-scores2

        $ bob measure det --test -o my_det.pdf dev-scores1 test-scores1
    """
    process = figure.Det(ctx, scores, test, load.split_files)
    process.run()

@click.command()
@common_options.scores_argument(test_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.axis_fontsize_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kargs):
    """Plot EPC (expected performance curve):
    plots the error rate on the test set depending on a threshold selected
    a-priori on the development set and accounts for varying relative cost β
    ∈ [0; 1] of FPR and FNR when calculating the threshold.

    You need provide one or more development score and test file(s)
    for each experiment.

    Examples:
        $ bob measure epc dev-scores test-scores

        $ bob measure epc -o my_epc.pdf dev-scores1 test-scores1
    """
    process = figure.Epc(ctx, scores, True, load.split_files)
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.test_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.threshold_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, test, **kargs):
    """ Plots histograms of positive and negatives along with threshold
    criterion.

    You need provide one or more development score file(s) for each experiment.
    You can also provide test files along with dev files but the flag `--test`
    is required in that case.

    Examples:
        $ bob measure hist dev-scores

        $ bob measure hist --test dev-scores1 test-scores1 dev-scores2
        test-scores2

        $ bob measure hist --test --criter hter dev-scores1 test-scores1
    """
    process = figure.Hist(ctx, scores, test, load.split_files)
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.titles_option()
@common_options.sep_dev_test_option()
@common_options.table_option()
@common_options.test_option()
@common_options.output_plot_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, test, **kargs):
    '''Runs error analysis on score sets
    1. Computes the threshold using either EER or min. HTER criteria on
        development set scores
    2. Applies the above threshold on test set scores to compute the HTER, if a
        test-score set is provided
    3. Reports error rates on the console
    4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
        file (unless --no-plot is passed)


    You need to provide 2 score files for each biometric system in this order:
    \b
    * development scores
    * evaluation scores

    Examples:
        $ bob measure evaluate dev-scores

        $ bob measure evaluate -t -l metrics.txt -o my_plots.pdf dev-scores test-scores
    '''
    #first time erase if existing file
    click.echo("Computing metrics with EER...")
    ctx.meta['criter'] = 'eer' #no criterion passed to evaluate
    ctx.invoke(metrics, scores=scores, test=test)
    #second time, appends the content
    click.echo("Computing metrics with HTER...")
    ctx.meta['criter'] = 'hter' #no criterion passed in evaluate
    ctx.invoke(metrics, scores=scores, test=test)
    if 'log' in ctx.meta:
        click.echo("[metrics] => %s" % ctx.meta['log'])

    #avoid closing pdf file before all figures are plotted
    ctx.meta['closef'] = False
    if test:
        click.echo("Starting evaluate with dev and test scores...")
    else:
        click.echo("Starting evaluate with dev scores only...")
    click.echo("Computing ROC...")
    ctx.forward(roc)
    click.echo("Computing DET...")
    ctx.forward(det)
    if test:
        click.echo("Computing EPC...")
        ctx.forward(epc)
    #the last one closes the file
    ctx.meta['closef'] = True
    click.echo("Computing score histograms...")
    ctx.forward(hist)
    click.echo("Evaluate successfully completed!")
    click.echo("[plots] => %s" % (ctx.meta['output']))