pad_commands.py 7.28 KB
Newer Older
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
1
"""The main entry for bob pad commands.
2 3 4
"""
import click
from bob.measure.script import common_options
5
from bob.extension.scripts.click_helper import verbosity_option
6 7 8
import bob.bio.base.script.gen as bio_gen
import bob.measure.script.figure as measure_figure
from bob.bio.base.score import load
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
9
from . import pad_figure as figure
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
10 11
from .error_utils import negatives_per_pai_and_positives
from functools import partial
12

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
13
SCORE_FORMAT = (
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
    "Files must be 4-col format, see " ":py:func:`bob.bio.base.score.load.four_column`."
)
CRITERIA = (
    "eer",
    "min-hter",
    "far",
    "bpcer5000",
    "bpcer2000",
    "bpcer1000",
    "bpcer500",
    "bpcer200",
    "bpcer100",
    "bpcer50",
    "bpcer20",
    "bpcer10",
    "bpcer5",
    "bpcer2",
    "bpcer1",
Anjith GEORGE's avatar
Anjith GEORGE committed
32 33 34 35 36 37 38 39 40 41 42 43
    "apcer5000",
    "apcer2000",
    "apcer1000",
    "apcer500",
    "apcer200",
    "apcer100",
    "apcer50",
    "apcer20",
    "apcer10",
    "apcer5",
    "apcer2",
    "apcer1",
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
44 45 46 47 48 49 50 51 52
)


def metrics_option(
    sname="-m",
    lname="--metrics",
    name="metrics",
    help="List of metrics to print. Provide a string with comma separated metric "
    "names. For possible values see the default value.",
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
53
    default="apcer_pais,apcer_ap,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score",
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
    **kwargs
):
    """The metrics option"""

    def custom_metrics_option(func):
        def callback(ctx, param, value):
            if value is not None:
                value = value.split(",")
            ctx.meta[name] = value
            return value

        return click.option(
            sname,
            lname,
            default=default,
            help=help,
            show_default=True,
            callback=callback,
            **kwargs
        )(func)

    return custom_metrics_option


def regexps_option(
    help="A list of regular expressions (by repeating this option) to be used to "
    "categorize PAIs. Each regexp must match one type of PAI.",
    **kwargs
):
    def custom_regexps_option(func):
        def callback(ctx, param, value):
            ctx.meta["regexps"] = value
            return value

        return click.option(
            "-r",
            "--regexps",
            default=None,
            multiple=True,
            help=help,
            callback=callback,
            **kwargs
        )(func)

    return custom_regexps_option


def regexp_column_option(
    help="The column in the score files to match the regular expressions against.",
    **kwargs
):
    def custom_regexp_column_option(func):
        def callback(ctx, param, value):
            ctx.meta["regexp_column"] = value
            return value

        return click.option(
            "-rc",
            "--regexp-column",
            default="real_id",
            type=click.Choice(("claimed_id", "real_id", "test_label")),
            help=help,
            show_default=True,
            callback=callback,
            **kwargs
        )(func)

    return custom_regexp_column_option
122 123


124
@click.command()
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
125 126 127 128 129 130
@click.argument("outdir")
@click.option("-mm", "--mean-match", default=10, type=click.FLOAT, show_default=True)
@click.option(
    "-mnm", "--mean-non-match", default=-10, type=click.FLOAT, show_default=True
)
@click.option("-n", "--n-sys", default=1, type=click.INT, show_default=True)
131 132
@verbosity_option()
@click.pass_context
133
def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
134 135 136 137 138 139 140 141 142 143 144 145
    """Generate random scores.
    Generates random scores in 4col or 5col format. The scores are generated
    using Gaussian distribution whose mean is an input
    parameter. The generated scores can be used as hypothetical datasets.
    Invokes :py:func:`bob.bio.base.script.commands.gen`.
    """
    ctx.meta["five_col"] = False
    ctx.forward(bio_gen.gen)


@common_options.metrics_command(
    common_options.METRICS_HELP.format(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
146
        names="FtA, APCER_AP, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
147 148
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
149
        hter_note="Note that APCER_AP = max(APCER_pais), BPCER=FNR, "
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
150 151
        "FAR = FPR * (1 - FtA), "
        "FRR = FtA + FNR * (1 - FtA), "
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
152
        "ACER = (APCER_AP + BPCER) / 2, "
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
153 154 155 156 157 158 159 160 161 162
        "and HTER = (FPR + FNR) / 2. "
        "You can control which metrics are printed using the --metrics option. "
        "You can use --regexps and --regexp_column options to change the behavior "
        "of finding Presentation Attack Instrument (PAI) types",
        command="bob pad metrics",
    ),
    criteria=CRITERIA,
    epilog="""\b
More Examples:
\b
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
163
bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer_ap,bpcer,acer,hter \
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
164 165 166 167 168 169 170 171 172 173 174 175 176 177
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}

See also ``bob pad multi-metrics``.
""",
)
@regexps_option()
@regexp_column_option()
@metrics_option()
def metrics(ctx, scores, evaluation, regexps, regexp_column, metrics, **kwargs):
    load_fn = partial(
        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
    )
    process = figure.Metrics(ctx, scores, evaluation, load_fn, metrics)
    process.run()
178 179


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
180
@common_options.roc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
181 182
    common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob pad roc")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
183
def roc(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
184 185
    process = figure.Roc(ctx, scores, evaluation, load.split)
    process.run()
186 187


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
188
@common_options.det_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
189 190
    common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob pad det")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
191
def det(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
192 193
    process = figure.Det(ctx, scores, evaluation, load.split)
    process.run()
194 195


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
196
@common_options.epc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
197 198
    common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob pad epc")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
199
def epc(ctx, scores, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
200 201
    process = measure_figure.Epc(ctx, scores, True, load.split, hter="ACER")
    process.run()
202 203


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
204
@common_options.hist_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
205 206
    common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob pad hist")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
207
def hist(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
208 209
    process = figure.Hist(ctx, scores, evaluation, load.split)
    process.run()
210 211


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
212 213
@common_options.evaluate_command(
    common_options.EVALUATE_HELP.format(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
214 215 216 217
        score_format=SCORE_FORMAT, command="bob pad evaluate"
    ),
    criteria=CRITERIA,
)
218
def evaluate(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
219 220 221
    common_options.evaluate_flow(
        ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
    )
222 223 224 225


@common_options.multi_metrics_command(
    common_options.MULTI_METRICS_HELP.format(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
        names="FtA, APCER, BPCER, FAR, FRR, ACER, HTER, precision, recall, f1_score",
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
        command="bob pad multi-metrics",
    ),
    criteria=CRITERIA,
    epilog="""\b
More examples:

\b
bob pad multi-metrics -vvv -e -pn 6 -lg IQM,LBP -r print -r video \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_3_{1,2,3,4,5,6}/scores/scores-{dev,eval}

See also ``bob pad metrics``.
""",
)
@regexps_option()
@regexp_column_option()
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
244
@metrics_option(default="fta,apcer_pais,apcer_ap,bpcer,acer,hter")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
245 246 247 248 249 250 251 252 253
def multi_metrics(
    ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs
):
    ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
    load_fn = partial(
        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
    )
    process = figure.MultiMetrics(ctx, scores, evaluation, load_fn, metrics)
    process.run()