commands.py 7.27 KB
Newer Older
1
""" Click commands for ``bob.bio.base`` """
2
3

import click
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
4

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
5
import bob.measure.script.figure as measure_figure
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
6

7
from bob.extension.scripts.click_helper import verbosity_option
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
8
from bob.measure.script import common_options
9

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
10
11
from ..score import load
from . import figure as bio_figure
12
13
14
15

SCORE_FORMAT = (
    "Files must be 4- or 5- columns format, see "
    ":py:func:`bob.bio.base.score.load.four_column` and "
16
17
18
    ":py:func:`bob.bio.base.score.load.five_column` for details."
)
CRITERIA = ("eer", "min-hter", "far", "mindcf", "cllr", "rr")
19
20


21
def rank_option(**kwargs):
22
23
    """Get option for rank parameter"""

24
25
26
    def custom_rank_option(func):
        def callback(ctx, param, value):
            value = 1 if value < 0 else value
27
            ctx.meta["rank"] = value
28
            return value
29

30
        return click.option(
31
32
33
34
35
36
37
38
39
40
            "-rk",
            "--rank",
            type=click.INT,
            default=1,
            help="Provide rank for the command",
            callback=callback,
            show_default=True,
            **kwargs
        )(func)

41
42
    return custom_rank_option

43

44
45
@common_options.metrics_command(
    common_options.METRICS_HELP.format(
46
        names="FtA, FAR, FRR, FMR, FNMR, HTER",
47
48
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
49
50
        hter_note="Note that FAR = FMR * (1 - FtA), FRR = FtA + FNMR * (1 - FtA) "
        "and HTER = (FMR + FNMR) / 2",
51
52
53
54
        command="bob bio metrics",
    ),
    criteria=CRITERIA,
)
55
@common_options.cost_option()
56
def metrics(ctx, scores, evaluation, **kwargs):
57
    if "criterion" in ctx.meta and ctx.meta["criterion"] == "rr":
58
        process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc)
59
    else:
60
        process = bio_figure.Metrics(ctx, scores, evaluation, load.split)
61
62
    process.run()

63

64
@common_options.roc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
65
66
67
    common_options.ROC_HELP.format(
        score_format=SCORE_FORMAT, command="bob bio roc"
    )
68
)
69
def roc(ctx, scores, evaluation, **kwargs):
70
    process = bio_figure.Roc(ctx, scores, evaluation, load.split)
71
72
    process.run()

73

74
@common_options.det_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
75
76
77
    common_options.DET_HELP.format(
        score_format=SCORE_FORMAT, command="bob bio det"
    )
78
)
79
def det(ctx, scores, evaluation, **kwargs):
80
    process = bio_figure.Det(ctx, scores, evaluation, load.split)
81
82
    process.run()

83

84
@common_options.epc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
85
86
87
    common_options.EPC_HELP.format(
        score_format=SCORE_FORMAT, command="bob bio epc"
    )
88
)
89
def epc(ctx, scores, **kwargs):
90
    process = measure_figure.Epc(ctx, scores, True, load.split)
91
    process.run()
92
93


94
@common_options.hist_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
95
96
97
    common_options.HIST_HELP.format(
        score_format=SCORE_FORMAT, command="bob bio hist"
    )
98
)
99
def hist(ctx, scores, evaluation, **kwargs):
100
    process = bio_figure.Hist(ctx, scores, evaluation, load.split)
101
102
    process.run()

103

104
105
@common_options.evaluate_command(
    common_options.EVALUATE_HELP.format(
106
107
108
109
        score_format=SCORE_FORMAT, command="bob bio evaluate"
    ),
    criteria=CRITERIA,
)
110
111
112
@common_options.cost_option()
def evaluate(ctx, scores, evaluation, **kwargs):
    common_options.evaluate_flow(
113
114
        ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
    )
115
116


117
118
@common_options.multi_metrics_command(
    common_options.MULTI_METRICS_HELP.format(
119
        names="FtA, FAR, FRR, FMR, FNMR, HTER",
120
121
122
123
124
125
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
        command="bob bio multi-metrics",
    ),
    criteria=CRITERIA,
)
126
def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
127
    ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
128
    process = bio_figure.MultiMetrics(ctx, scores, evaluation, load.split)
129
    process.run()
130
131


Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
132
133
@click.command()
@common_options.scores_argument(nargs=-1)
134
@common_options.titles_option()
135
@common_options.legends_option()
136
@common_options.sep_dev_eval_option()
137
@common_options.output_plot_file_option(default_out="cmc.pdf")
138
@common_options.eval_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
139
140
141
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
@common_options.x_rotation_option()
142
@common_options.const_layout_option()
143
@common_options.style_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
144
@common_options.linestyles_option()
145
@common_options.figsize_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
146
147
@verbosity_option()
@click.pass_context
148
def cmc(ctx, scores, evaluation, **kwargs):
149
    """Plot CMC (cumulative match characteristic curve).
150
151
152
153
154
155
    graphical presentation of results of an identification task eval, plotting
    rank values on the x-axis and the probability of correct identification at
    or below that rank on the y-axis. The values for the axis will be computed
    using :py:func:`bob.measure.cmc`.

    You need to provide one or more development score file(s) for each
156
157
    experiment. You can also provide eval files along with dev files. If
    eval-scores are used, the flag `--eval` must be used. is required
158
    in that case. Files must be 4- or 5- columns format, see
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
159
160
161
162
163
    :py:func:`bob.bio.base.score.load.four_column` and
    :py:func:`bob.bio.base.score.load.five_column` for details.


    Examples:
164
        $ bob bio cmc -v dev-scores
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
165

166
        $ bob bio cmc -v dev-scores1 eval-scores1 dev-scores2
167
        eval-scores2
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
168

169
        $ bob bio cmc -v -o my_roc.pdf dev-scores1 eval-scores1
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
170
    """
171
    process = bio_figure.Cmc(ctx, scores, evaluation, load.cmc)
172
    process.run()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
173

174

175
@click.command()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
176
@common_options.scores_argument(nargs=-1)
177
@common_options.titles_option()
178
@common_options.legends_option()
179
@common_options.sep_dev_eval_option()
180
@common_options.output_plot_file_option(default_out="dir.pdf")
181
@common_options.eval_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
182
183
184
@common_options.semilogx_option(True)
@common_options.axes_val_option(dflt=None)
@common_options.x_rotation_option()
185
@rank_option()
186
@common_options.const_layout_option()
187
@common_options.style_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
188
@common_options.linestyles_option()
189
@common_options.figsize_option()
190
@common_options.min_far_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
191
192
@verbosity_option()
@click.pass_context
193
def dir(ctx, scores, evaluation, **kwargs):
194
    """Plots the Detection & Identification Rate curve over the FAR.
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
195

196
197
198
    This curve is designed to be used in an open set identification protocol,
    and defined in Chapter 14.1 of [LiJain2005]_.  It requires to have at least
    one open set probe item, i.e., with no corresponding gallery, such that the
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
199
200
    positives for that pair are ``None``.

201
202
    The detection and identification curve first computes FAR thresholds based
    on the out-of-set probe scores (negative scores).  For each probe item, the
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
203
204
205
206
207
208
    **maximum** negative score is used.  Then, it plots the detection and
    identification rates for those thresholds, which are based on the in-set
    probe scores only. See [LiJain2005]_ for more details.

    .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005

209
    You need to provide one or more development score file(s) for each
210
211
    experiment. You can also provide eval files along with dev files. If
    eval-scores are used, the flag `--eval` must be used. is required
212
    in that case. Files must be 4- or 5- columns format, see
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
213
214
215
216
    :py:func:`bob.bio.base.score.load.four_column` and
    :py:func:`bob.bio.base.score.load.five_column` for details.

    Examples:
217
        $ bob bio dir -e -v dev-scores
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
218

219
        $ bob bio dir -v dev-scores1 eval-scores1 dev-scores2
220
        eval-scores2
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
221

222
        $ bob bio dir -v -o my_roc.pdf dev-scores1 eval-scores1
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
223
    """
224
    process = bio_figure.Dir(ctx, scores, evaluation, load.cmc)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
225
    process.run()