pad_commands.py 7.08 KB
Newer Older
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
1
"""The main entry for bob pad commands.
2 3 4
"""
import click
from bob.measure.script import common_options
5
from bob.extension.scripts.click_helper import verbosity_option
6 7 8
import bob.bio.base.script.gen as bio_gen
import bob.measure.script.figure as measure_figure
from bob.bio.base.score import load
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
9
from . import pad_figure as figure
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
10 11
from .error_utils import negatives_per_pai_and_positives
from functools import partial
12

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
13
SCORE_FORMAT = (
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
    "Files must be 4-col format, see " ":py:func:`bob.bio.base.score.load.four_column`."
)
CRITERIA = (
    "eer",
    "min-hter",
    "far",
    "bpcer5000",
    "bpcer2000",
    "bpcer1000",
    "bpcer500",
    "bpcer200",
    "bpcer100",
    "bpcer50",
    "bpcer20",
    "bpcer10",
    "bpcer5",
    "bpcer2",
    "bpcer1",
)


def metrics_option(
    sname="-m",
    lname="--metrics",
    name="metrics",
    help="List of metrics to print. Provide a string with comma separated metric "
    "names. For possible values see the default value.",
    default="apcer_pais,apcer,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score",
    **kwargs
):
    """The metrics option"""

    def custom_metrics_option(func):
        def callback(ctx, param, value):
            if value is not None:
                value = value.split(",")
            ctx.meta[name] = value
            return value

        return click.option(
            sname,
            lname,
            default=default,
            help=help,
            show_default=True,
            callback=callback,
            **kwargs
        )(func)

    return custom_metrics_option


def regexps_option(
    help="A list of regular expressions (by repeating this option) to be used to "
    "categorize PAIs. Each regexp must match one type of PAI.",
    **kwargs
):
    def custom_regexps_option(func):
        def callback(ctx, param, value):
            ctx.meta["regexps"] = value
            return value

        return click.option(
            "-r",
            "--regexps",
            default=None,
            multiple=True,
            help=help,
            callback=callback,
            **kwargs
        )(func)

    return custom_regexps_option


def regexp_column_option(
    help="The column in the score files to match the regular expressions against.",
    **kwargs
):
    def custom_regexp_column_option(func):
        def callback(ctx, param, value):
            ctx.meta["regexp_column"] = value
            return value

        return click.option(
            "-rc",
            "--regexp-column",
            default="real_id",
            type=click.Choice(("claimed_id", "real_id", "test_label")),
            help=help,
            show_default=True,
            callback=callback,
            **kwargs
        )(func)

    return custom_regexp_column_option
110 111


112
@click.command()
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
113 114 115 116 117 118
@click.argument("outdir")
@click.option("-mm", "--mean-match", default=10, type=click.FLOAT, show_default=True)
@click.option(
    "-mnm", "--mean-non-match", default=-10, type=click.FLOAT, show_default=True
)
@click.option("-n", "--n-sys", default=1, type=click.INT, show_default=True)
119 120
@verbosity_option()
@click.pass_context
121
def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
    """Generate random scores.
    Generates random scores in 4col or 5col format. The scores are generated
    using Gaussian distribution whose mean is an input
    parameter. The generated scores can be used as hypothetical datasets.
    Invokes :py:func:`bob.bio.base.script.commands.gen`.
    """
    ctx.meta["five_col"] = False
    ctx.forward(bio_gen.gen)


@common_options.metrics_command(
    common_options.METRICS_HELP.format(
        names="FtA, APCER, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
        hter_note="Note that APCER = max(APCER_pais), BPCER=FNR, "
        "FAR = FPR * (1 - FtA), "
        "FRR = FtA + FNR * (1 - FtA), "
        "ACER = (APCER + BPCER) / 2, "
        "and HTER = (FPR + FNR) / 2. "
        "You can control which metrics are printed using the --metrics option. "
        "You can use --regexps and --regexp_column options to change the behavior "
        "of finding Presentation Attack Instrument (PAI) types",
        command="bob pad metrics",
    ),
    criteria=CRITERIA,
    epilog="""\b
More Examples:
\b
bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer,bpcer,acer,hter \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}

See also ``bob pad multi-metrics``.
""",
)
@regexps_option()
@regexp_column_option()
@metrics_option()
def metrics(ctx, scores, evaluation, regexps, regexp_column, metrics, **kwargs):
    load_fn = partial(
        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
    )
    process = figure.Metrics(ctx, scores, evaluation, load_fn, metrics)
    process.run()
166 167


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
168
@common_options.roc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
169 170
    common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob pad roc")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
171
def roc(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
172 173
    process = figure.Roc(ctx, scores, evaluation, load.split)
    process.run()
174 175


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
176
@common_options.det_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
177 178
    common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob pad det")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
179
def det(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
180 181
    process = figure.Det(ctx, scores, evaluation, load.split)
    process.run()
182 183


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
184
@common_options.epc_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
185 186
    common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob pad epc")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
187
def epc(ctx, scores, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
188 189
    process = measure_figure.Epc(ctx, scores, True, load.split, hter="ACER")
    process.run()
190 191


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
192
@common_options.hist_command(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
193 194
    common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob pad hist")
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
195
def hist(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
196 197
    process = figure.Hist(ctx, scores, evaluation, load.split)
    process.run()
198 199


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
200 201
@common_options.evaluate_command(
    common_options.EVALUATE_HELP.format(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
202 203 204 205
        score_format=SCORE_FORMAT, command="bob pad evaluate"
    ),
    criteria=CRITERIA,
)
206
def evaluate(ctx, scores, evaluation, **kwargs):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
207 208 209
    common_options.evaluate_flow(
        ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
    )
210 211 212 213


@common_options.multi_metrics_command(
    common_options.MULTI_METRICS_HELP.format(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
        names="FtA, APCER, BPCER, FAR, FRR, ACER, HTER, precision, recall, f1_score",
        criteria=CRITERIA,
        score_format=SCORE_FORMAT,
        command="bob pad multi-metrics",
    ),
    criteria=CRITERIA,
    epilog="""\b
More examples:

\b
bob pad multi-metrics -vvv -e -pn 6 -lg IQM,LBP -r print -r video \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_3_{1,2,3,4,5,6}/scores/scores-{dev,eval}

See also ``bob pad metrics``.
""",
)
@regexps_option()
@regexp_column_option()
@metrics_option(default="fta,apcer_pais,apcer,bpcer,acer,hter")
def multi_metrics(
    ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs
):
    ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
    load_fn = partial(
        negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
    )
    process = figure.MultiMetrics(ctx, scores, evaluation, load_fn, metrics)
    process.run()