commands.py 9.69 KB
Newer Older
1 2 3 4 5 6 7
''' Click commands for ``bob.measure`` '''


import click
from .. import load
from . import figure
from . import common_options
8 9
from bob.extension.scripts.click_helper import (verbosity_option,
                                                open_file_mode_option)
10

11 12
@click.command()
@common_options.scores_argument(nargs=-1)
13
@common_options.eval_option()
14
@common_options.table_option()
15
@common_options.output_log_metric_option()
16
@common_options.criterion_option()
17 18
@common_options.thresholds_option()
@common_options.far_option()
19
@common_options.legends_option()
20
@open_file_mode_option()
21 22
@verbosity_option()
@click.pass_context
23
def metrics(ctx, scores, evaluation, **kwargs):
24
    """Prints a table that contains FtA, FAR, FRR, FMR, FMNR, HTER for a given
25
    threshold criterion (eer or min-hter).
26

27
    You need to provide one or more development score file(s) for each experiment.
28 29
    You can also provide evaluation files along with dev files. If only dev scores
    are provided, you must use flag `--no-evaluation`.
30

31
    Resulting table format can be changed using the `--tablefmt`.
32 33 34 35

    Examples:
        $ bob measure metrics dev-scores

36
        $ bob measure metrics -l results.txt dev-scores1 eval-scores1
37

38
        $ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
39
    """
40
    process = figure.Metrics(ctx, scores, evaluation, load.split)
41 42 43 44
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
45
@common_options.title_option()
46
@common_options.legends_option()
47
@common_options.sep_dev_eval_option()
48
@common_options.output_plot_file_option(default_out='roc.pdf')
49
@common_options.eval_option()
50
@common_options.points_curve_option()
51
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
52
@common_options.min_far_option()
53
@common_options.x_rotation_option()
54 55 56
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.lines_at_option()
57
@common_options.const_layout_option()
58 59
@common_options.figsize_option()
@common_options.style_option()
60
@common_options.linestyles_option()
61 62
@verbosity_option()
@click.pass_context
63
def roc(ctx, scores, evaluation, **kwargs):
64 65 66 67 68
    """Plot ROC (receiver operating characteristic) curve:
    The plot will represent the false match rate on the horizontal axis and the
    false non match rate on the vertical axis.  The values for the axis will be
    computed using :py:func:`bob.measure.roc`.

69
    You need to provide one or more development score file(s) for each experiment.
70 71
    You can also provide evaluation files along with dev files. If only dev scores
    are provided, you must use flag `--no-evaluation`.
72 73 74 75

    Examples:
        $ bob measure roc dev-scores

76 77
        $ bob measure roc dev-scores1 eval-scores1 dev-scores2
        eval-scores2
78

79
        $ bob measure roc -o my_roc.pdf dev-scores1 eval-scores1
80
    """
81
    process = figure.Roc(ctx, scores, evaluation, load.split)
82 83 84 85 86
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
87
@common_options.title_option()
88
@common_options.legends_option()
89 90
@common_options.sep_dev_eval_option()
@common_options.eval_option()
91
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
92
@common_options.min_far_option()
93
@common_options.x_rotation_option(dflt=45)
94 95
@common_options.x_label_option()
@common_options.y_label_option()
96
@common_options.points_curve_option()
97
@common_options.lines_at_option()
98
@common_options.const_layout_option()
99 100
@common_options.figsize_option()
@common_options.style_option()
101
@common_options.linestyles_option()
102 103
@verbosity_option()
@click.pass_context
104
def det(ctx, scores, evaluation, **kwargs):
105 106 107 108
    """Plot DET (detection error trade-off) curve:
    modified ROC curve which plots error rates on both axes
    (false positives on the x-axis and false negatives on the y-axis)

109
    You need to provide one or more development score file(s) for each experiment.
110 111
    You can also provide evaluation files along with dev files. If only dev scores
    are provided, you must use flag `--no-evaluation`.
112 113 114 115

    Examples:
        $ bob measure det dev-scores

116 117
        $ bob measure det dev-scores1 eval-scores1 dev-scores2
        eval-scores2
118

119
        $ bob measure det -o my_det.pdf dev-scores1 eval-scores1
120
    """
121
    process = figure.Det(ctx, scores, evaluation, load.split)
122 123 124
    process.run()

@click.command()
125
@common_options.scores_argument(min_arg=1, force_eval=True, nargs=-1)
126
@common_options.output_plot_file_option(default_out='epc.pdf')
127
@common_options.title_option()
128
@common_options.legends_option()
129
@common_options.points_curve_option()
130
@common_options.const_layout_option()
131 132
@common_options.x_label_option()
@common_options.y_label_option()
133 134
@common_options.figsize_option()
@common_options.style_option()
135
@common_options.linestyles_option()
136 137
@verbosity_option()
@click.pass_context
138
def epc(ctx, scores, **kwargs):
139
    """Plot EPC (expected performance curve):
140
    plots the error rate on the eval set depending on a threshold selected
141 142
    a-priori on the development set and accounts for varying relative cost
    in [0; 1] of FPR and FNR when calculating the threshold.
143

144
    You need to provide one or more development score and eval file(s)
145 146 147
    for each experiment.

    Examples:
148
        $ bob measure epc dev-scores eval-scores
149

150
        $ bob measure epc -o my_epc.pdf dev-scores1 eval-scores1
151
    """
152
    process = figure.Epc(ctx, scores, True, load.split)
153 154 155 156 157
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
158
@common_options.eval_option()
159 160
@common_options.n_bins_option()
@common_options.criterion_option()
161
@common_options.thresholds_option()
162 163
@common_options.const_layout_option()
@common_options.print_filenames_option()
164
@common_options.title_option()
165
@common_options.legends_option()
166
@common_options.figsize_option(dflt=None)
167
@common_options.style_option()
168
@common_options.linestyles_option()
169 170
@common_options.subplot_option()
@common_options.legend_ncols_option()
171 172
@verbosity_option()
@click.pass_context
173
def hist(ctx, scores, evaluation, **kwargs):
174 175 176
    """ Plots histograms of positive and negatives along with threshold
    criterion.

177
    You need to provide one or more development score file(s) for each experiment.
178 179 180
    You can also provide evaluation files along with dev files. If only dev scores
    are provided, you must use flag `--no-evaluation`.

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
181
    By default, when eval-scores are given, only eval-scores histograms are
182
    displayed with threshold line
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
183
    computed from dev-scores. If you want to display dev-scores distributions
184
    as well, use ``--show-dev`` option.
185 186 187 188

    Examples:
        $ bob measure hist dev-scores

189 190
        $ bob measure hist dev-scores1 eval-scores1 dev-scores2
        eval-scores2
191

192
        $ bob measure hist --criterion min-hter --show-dev dev-scores1 eval-scores1
193
    """
194
    process = figure.Hist(ctx, scores, evaluation, load.split)
195 196 197 198
    process.run()

@click.command()
@common_options.scores_argument(nargs=-1)
199
@common_options.legends_option()
200
@common_options.sep_dev_eval_option()
201
@common_options.table_option()
202
@common_options.eval_option()
203
@common_options.output_log_metric_option()
204 205 206
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.n_bins_option()
207
@common_options.lines_at_option()
208
@common_options.const_layout_option()
209 210
@common_options.figsize_option()
@common_options.style_option()
211
@common_options.linestyles_option()
212 213
@verbosity_option()
@click.pass_context
214
def evaluate(ctx, scores, evaluation, **kwargs):
215
    '''Runs error analysis on score sets
216 217

    \b
218
    1. Computes the threshold using either EER or min. HTER criteria on
219
       development set scores
220 221
    2. Applies the above threshold on evaluation set scores to compute the HTER, if a
       eval-score set is provided
222 223
    3. Reports error rates on the console
    4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
224
       file
225 226 227


    You need to provide 2 score files for each biometric system in this order:
228

229 230 231 232 233 234 235
    \b
    * development scores
    * evaluation scores

    Examples:
        $ bob measure evaluate dev-scores

236 237 238 239 240
        $ bob measure evaluate scores-dev1 scores-eval1 scores-dev2
        scores-eval2

        $ bob measure evaluate /path/to/sys-{1,2,3}/scores-{dev,eval}

241
        $ bob measure evaluate -l metrics.txt -o my_plots.pdf dev-scores eval-scores
242
    '''
243
    # first time erase if existing file
244
    ctx.meta['open_mode'] = 'w'
245
    click.echo("Computing metrics with EER...")
246
    ctx.meta['criterion'] = 'eer'  # no criterion passed to evaluate
247
    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
248
    # second time, appends the content
249
    ctx.meta['open_mode'] = 'a'
250 251
    click.echo("Computing metrics with min-HTER...")
    ctx.meta['criterion'] = 'min-hter'  # no criterion passed in evaluate
252
    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
253 254 255
    if 'log' in ctx.meta:
        click.echo("[metrics] => %s" % ctx.meta['log'])

256
    # avoid closing pdf file before all figures are plotted
257
    ctx.meta['closef'] = False
258 259
    if evaluation:
        click.echo("Starting evaluate with dev and eval scores...")
260 261 262
    else:
        click.echo("Starting evaluate with dev scores only...")
    click.echo("Computing ROC...")
263 264
    # set axes limits for ROC
    ctx.forward(roc) # use class defaults plot settings
265
    click.echo("Computing DET...")
266
    ctx.forward(det) # use class defaults plot settings
267
    if evaluation:
268
        click.echo("Computing EPC...")
269 270
        ctx.forward(epc) # use class defaults plot settings
    # the last one closes the file
271 272
    ctx.meta['closef'] = True
    click.echo("Computing score histograms...")
273
    ctx.meta['criterion'] = 'eer'  # no criterion passed in evaluate
274 275 276
    ctx.forward(hist)
    click.echo("Evaluate successfully completed!")
    click.echo("[plots] => %s" % (ctx.meta['output']))