commands.py 9.71 KB
Newer Older
1
2
3
4
5
6
7
''' Click commands for ``bob.measure`` '''


import click
from .. import load
from . import figure
from . import common_options
8
9
from bob.extension.scripts.click_helper import (verbosity_option,
                                                open_file_mode_option)
10

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
11

12
13
@click.command()
@common_options.scores_argument(nargs=-1)
14
@common_options.eval_option()
15
@common_options.table_option()
16
@common_options.output_log_metric_option()
17
@common_options.criterion_option()
18
19
@common_options.thresholds_option()
@common_options.far_option()
20
@common_options.legends_option()
21
@open_file_mode_option()
22
23
@verbosity_option()
@click.pass_context
24
def metrics(ctx, scores, evaluation, **kwargs):
25
    """Prints a table that contains FtA, FAR, FRR, FMR, FMNR, HTER for a given
26
    threshold criterion (eer or min-hter).
27

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
28
29
30
    You need to provide one or more development score file(s) for each
    experiment. You can also provide evaluation files along with dev files. If
    only dev scores are provided, you must use flag `--no-evaluation`.
31

32
    Resulting table format can be changed using the `--tablefmt`.
33
34
35
36

    Examples:
        $ bob measure metrics dev-scores

37
        $ bob measure metrics -l results.txt dev-scores1 eval-scores1
38

39
        $ bob measure metrics {dev,eval}-scores1 {dev,eval}-scores2
40
    """
41
    process = figure.Metrics(ctx, scores, evaluation, load.split)
42
43
    process.run()

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
44

45
46
@click.command()
@common_options.scores_argument(nargs=-1)
47
@common_options.title_option()
48
@common_options.legends_option()
49
@common_options.sep_dev_eval_option()
50
@common_options.output_plot_file_option(default_out='roc.pdf')
51
@common_options.eval_option()
52
@common_options.points_curve_option()
53
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
54
@common_options.min_far_option()
55
@common_options.x_rotation_option()
56
57
58
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.lines_at_option()
59
@common_options.const_layout_option()
60
61
@common_options.figsize_option()
@common_options.style_option()
62
@common_options.linestyles_option()
63
64
@verbosity_option()
@click.pass_context
65
def roc(ctx, scores, evaluation, **kwargs):
66
67
68
69
70
    """Plot ROC (receiver operating characteristic) curve:
    The plot will represent the false match rate on the horizontal axis and the
    false non match rate on the vertical axis.  The values for the axis will be
    computed using :py:func:`bob.measure.roc`.

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
71
72
73
    You need to provide one or more development score file(s) for each
    experiment. You can also provide evaluation files along with dev files. If
    only dev scores are provided, you must use flag `--no-evaluation`.
74
75

    Examples:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
76
        $ bob measure roc -v dev-scores
77

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
78
        $ bob measure roc -v dev-scores1 eval-scores1 dev-scores2
79
        eval-scores2
80

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
81
        $ bob measure roc -v -o my_roc.pdf dev-scores1 eval-scores1
82
    """
83
    process = figure.Roc(ctx, scores, evaluation, load.split)
84
85
    process.run()

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
86

87
88
89
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
90
@common_options.title_option()
91
@common_options.legends_option()
92
93
@common_options.sep_dev_eval_option()
@common_options.eval_option()
94
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
95
@common_options.min_far_option()
96
@common_options.x_rotation_option(dflt=45)
97
98
@common_options.x_label_option()
@common_options.y_label_option()
99
@common_options.points_curve_option()
100
@common_options.lines_at_option()
101
@common_options.const_layout_option()
102
103
@common_options.figsize_option()
@common_options.style_option()
104
@common_options.linestyles_option()
105
106
@verbosity_option()
@click.pass_context
107
def det(ctx, scores, evaluation, **kwargs):
108
109
110
111
    """Plot DET (detection error trade-off) curve:
    modified ROC curve which plots error rates on both axes
    (false positives on the x-axis and false negatives on the y-axis)

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
112
113
114
    You need to provide one or more development score file(s) for each
    experiment. You can also provide evaluation files along with dev files. If
    only dev scores are provided, you must use flag `--no-evaluation`.
115
116

    Examples:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
117
        $ bob measure det -v dev-scores
118

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
119
        $ bob measure det -v dev-scores1 eval-scores1 dev-scores2
120
        eval-scores2
121

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
122
        $ bob measure det -v -o my_det.pdf dev-scores1 eval-scores1
123
    """
124
    process = figure.Det(ctx, scores, evaluation, load.split)
125
126
    process.run()

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
127

128
@click.command()
129
@common_options.scores_argument(min_arg=1, force_eval=True, nargs=-1)
130
@common_options.output_plot_file_option(default_out='epc.pdf')
131
@common_options.title_option()
132
@common_options.legends_option()
133
@common_options.points_curve_option()
134
@common_options.const_layout_option()
135
136
@common_options.x_label_option()
@common_options.y_label_option()
137
138
@common_options.figsize_option()
@common_options.style_option()
139
@common_options.linestyles_option()
140
141
@verbosity_option()
@click.pass_context
142
def epc(ctx, scores, **kwargs):
143
    """Plot EPC (expected performance curve):
144
    plots the error rate on the eval set depending on a threshold selected
145
146
    a-priori on the development set and accounts for varying relative cost
    in [0; 1] of FPR and FNR when calculating the threshold.
147

148
    You need to provide one or more development score and eval file(s)
149
150
151
    for each experiment.

    Examples:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
152
        $ bob measure epc -v dev-scores eval-scores
153

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
154
        $ bob measure epc -v -o my_epc.pdf dev-scores1 eval-scores1
155
    """
156
    process = figure.Epc(ctx, scores, True, load.split)
157
158
    process.run()

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
159

160
161
162
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
163
@common_options.eval_option()
164
165
@common_options.n_bins_option()
@common_options.criterion_option()
166
@common_options.thresholds_option()
167
168
@common_options.const_layout_option()
@common_options.print_filenames_option()
169
@common_options.title_option()
170
@common_options.legends_option()
171
@common_options.figsize_option(dflt=None)
172
@common_options.style_option()
173
@common_options.linestyles_option()
174
175
@common_options.subplot_option()
@common_options.legend_ncols_option()
176
177
@verbosity_option()
@click.pass_context
178
def hist(ctx, scores, evaluation, **kwargs):
179
180
181
    """ Plots histograms of positive and negatives along with threshold
    criterion.

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
182
183
184
    You need to provide one or more development score file(s) for each
    experiment. You can also provide evaluation files along with dev files. If
    only dev scores are provided, you must use flag `--no-evaluation`.
185

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
186
    By default, when eval-scores are given, only eval-scores histograms are
187
    displayed with threshold line
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
188
    computed from dev-scores. If you want to display dev-scores distributions
189
    as well, use ``--show-dev`` option.
190
191

    Examples:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
192
        $ bob measure hist -v dev-scores
193

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
194
        $ bob measure hist -v dev-scores1 eval-scores1 dev-scores2
195
        eval-scores2
196

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
197
        $ bob measure hist -v --criterion min-hter --show-dev dev-scores1 eval-scores1
198
    """
199
    process = figure.Hist(ctx, scores, evaluation, load.split)
200
201
    process.run()

Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
202

203
204
@click.command()
@common_options.scores_argument(nargs=-1)
205
@common_options.legends_option()
206
@common_options.sep_dev_eval_option()
207
@common_options.table_option()
208
@common_options.eval_option()
209
@common_options.output_log_metric_option()
210
211
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
212
@common_options.lines_at_option()
213
@common_options.const_layout_option()
214
215
@common_options.figsize_option()
@common_options.style_option()
216
@common_options.linestyles_option()
217
218
@verbosity_option()
@click.pass_context
219
def evaluate(ctx, scores, evaluation, **kwargs):
220
    '''Runs error analysis on score sets
221
222

    \b
223
    1. Computes the threshold using either EER or min. HTER criteria on
224
       development set scores
Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
225
226
    2. Applies the above threshold on evaluation set scores to compute the
       HTER, if a eval-score set is provided
227
228
    3. Reports error rates on the console
    4. Plots ROC, EPC, DET curves and score distributions to a multi-page PDF
229
       file
230
231
232


    You need to provide 2 score files for each biometric system in this order:
233

234
235
236
237
238
    \b
    * development scores
    * evaluation scores

    Examples:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
239
        $ bob measure evaluate -v dev-scores
240

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
241
        $ bob measure evaluate -v scores-dev1 scores-eval1 scores-dev2
242
243
        scores-eval2

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
244
        $ bob measure evaluate -v /path/to/sys-{1,2,3}/scores-{dev,eval}
245

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
246
        $ bob measure evaluate -v -l metrics.txt -o my_plots.pdf dev-scores eval-scores
247
    '''
248
    # first time erase if existing file
249
    ctx.meta['open_mode'] = 'w'
250
    click.echo("Computing metrics with EER...")
251
    ctx.meta['criterion'] = 'eer'  # no criterion passed to evaluate
252
    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
253
    # second time, appends the content
254
    ctx.meta['open_mode'] = 'a'
255
256
    click.echo("Computing metrics with min-HTER...")
    ctx.meta['criterion'] = 'min-hter'  # no criterion passed in evaluate
257
    ctx.invoke(metrics, scores=scores, evaluation=evaluation)
258
259
260
    if 'log' in ctx.meta:
        click.echo("[metrics] => %s" % ctx.meta['log'])

261
    # avoid closing pdf file before all figures are plotted
262
    ctx.meta['closef'] = False
263
264
    if evaluation:
        click.echo("Starting evaluate with dev and eval scores...")
265
266
267
    else:
        click.echo("Starting evaluate with dev scores only...")
    click.echo("Computing ROC...")
268
    # set axes limits for ROC
Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
269
    ctx.forward(roc)  # use class defaults plot settings
270
    click.echo("Computing DET...")
Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
271
    ctx.forward(det)  # use class defaults plot settings
272
    if evaluation:
273
        click.echo("Computing EPC...")
Amir MOHAMMADI's avatar
lint    
Amir MOHAMMADI committed
274
        ctx.forward(epc)  # use class defaults plot settings
275
    # the last one closes the file
276
277
    ctx.meta['closef'] = True
    click.echo("Computing score histograms...")
278
    ctx.meta['criterion'] = 'eer'  # no criterion passed in evaluate
279
280
281
    ctx.forward(hist)
    click.echo("Evaluate successfully completed!")
    click.echo("[plots] => %s" % (ctx.meta['output']))