Commit a772a3d5 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Compute APCER correctly

parent 9f28f82a
Pipeline #29681 passed with stage
in 14 minutes and 32 seconds
This diff is collapsed.
......@@ -7,20 +7,115 @@ import bob.bio.base.script.gen as bio_gen
import bob.measure.script.figure as measure_figure
from bob.bio.base.score import load
from . import pad_figure as figure
from .error_utils import negatives_per_pai_and_positives
from functools import partial
SCORE_FORMAT = (
"Files must be 4-col format, see "
":py:func:`bob.bio.base.score.load.four_column`.")
CRITERIA = ('eer', 'min-hter', 'bpcer20')
"Files must be 4-col format, see " ":py:func:`bob.bio.base.score.load.four_column`."
)
CRITERIA = (
"eer",
"min-hter",
"far",
"bpcer5000",
"bpcer2000",
"bpcer1000",
"bpcer500",
"bpcer200",
"bpcer100",
"bpcer50",
"bpcer20",
"bpcer10",
"bpcer5",
"bpcer2",
"bpcer1",
)
def metrics_option(
sname="-m",
lname="--metrics",
name="metrics",
help="List of metrics to print. Provide a string with comma separated metric "
"names. For possible values see the default value.",
default="apcer_pais,apcer,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score",
**kwargs
):
"""The metrics option"""
def custom_metrics_option(func):
def callback(ctx, param, value):
if value is not None:
value = value.split(",")
ctx.meta[name] = value
return value
return click.option(
sname,
lname,
default=default,
help=help,
show_default=True,
callback=callback,
**kwargs
)(func)
return custom_metrics_option
def regexps_option(
help="A list of regular expressions (by repeating this option) to be used to "
"categorize PAIs. Each regexp must match one type of PAI.",
**kwargs
):
def custom_regexps_option(func):
def callback(ctx, param, value):
ctx.meta["regexps"] = value
return value
return click.option(
"-r",
"--regexps",
default=None,
multiple=True,
help=help,
callback=callback,
**kwargs
)(func)
return custom_regexps_option
def regexp_column_option(
help="The column in the score files to match the regular expressions against.",
**kwargs
):
def custom_regexp_column_option(func):
def callback(ctx, param, value):
ctx.meta["regexp_column"] = value
return value
return click.option(
"-rc",
"--regexp-column",
default="real_id",
type=click.Choice(("claimed_id", "real_id", "test_label")),
help=help,
show_default=True,
callback=callback,
**kwargs
)(func)
return custom_regexp_column_option
@click.command()
@click.argument('outdir')
@click.option('-mm', '--mean-match', default=10, type=click.FLOAT,
show_default=True)
@click.option('-mnm', '--mean-non-match', default=-10,
type=click.FLOAT, show_default=True)
@click.option('-n', '--n-sys', default=1, type=click.INT, show_default=True)
@click.argument("outdir")
@click.option("-mm", "--mean-match", default=10, type=click.FLOAT, show_default=True)
@click.option(
"-mnm", "--mean-non-match", default=-10, type=click.FLOAT, show_default=True
)
@click.option("-n", "--n-sys", default=1, type=click.INT, show_default=True)
@verbosity_option()
@click.pass_context
def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
......@@ -30,48 +125,73 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
parameter. The generated scores can be used as hypothetical datasets.
Invokes :py:func:`bob.bio.base.script.commands.gen`.
"""
ctx.meta['five_col'] = False
ctx.meta["five_col"] = False
ctx.forward(bio_gen.gen)
@common_options.metrics_command(common_options.METRICS_HELP.format(
names='FtA, APCER, BPCER, FAR, FRR, ACER',
criteria=CRITERIA, score_format=SCORE_FORMAT,
hter_note='Note that FAR = APCER * (1 - FtA), '
'FRR = FtA + BPCER * (1 - FtA) and ACER = (APCER + BPCER) / 2.',
command='bob pad metrics'), criteria=CRITERIA)
def metrics(ctx, scores, evaluation, **kwargs):
process = figure.Metrics(ctx, scores, evaluation, load.split)
@common_options.metrics_command(
common_options.METRICS_HELP.format(
names="FtA, APCER, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
criteria=CRITERIA,
score_format=SCORE_FORMAT,
hter_note="Note that APCER = max(APCER_pais), BPCER=FNR, "
"FAR = FPR * (1 - FtA), "
"FRR = FtA + FNR * (1 - FtA), "
"ACER = (APCER + BPCER) / 2, "
"and HTER = (FPR + FNR) / 2. "
"You can control which metrics are printed using the --metrics option. "
"You can use --regexps and --regexp_column options to change the behavior "
"of finding Presentation Attack Instrument (PAI) types",
command="bob pad metrics",
),
criteria=CRITERIA,
epilog="""\b
More Examples:
\b
bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer,bpcer,acer,hter \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}
See also ``bob pad multi-metrics``.
""",
)
@regexps_option()
@regexp_column_option()
@metrics_option()
def metrics(ctx, scores, evaluation, regexps, regexp_column, metrics, **kwargs):
load_fn = partial(
negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
)
process = figure.Metrics(ctx, scores, evaluation, load_fn, metrics)
process.run()
@common_options.roc_command(
common_options.ROC_HELP.format(
score_format=SCORE_FORMAT, command='bob pad roc'))
common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob pad roc")
)
def roc(ctx, scores, evaluation, **kwargs):
process = figure.Roc(ctx, scores, evaluation, load.split)
process.run()
@common_options.det_command(
common_options.DET_HELP.format(
score_format=SCORE_FORMAT, command='bob pad det'))
common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob pad det")
)
def det(ctx, scores, evaluation, **kwargs):
process = figure.Det(ctx, scores, evaluation, load.split)
process.run()
@common_options.epc_command(
common_options.EPC_HELP.format(
score_format=SCORE_FORMAT, command='bob pad epc'))
common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob pad epc")
)
def epc(ctx, scores, **kwargs):
process = measure_figure.Epc(ctx, scores, True, load.split, hter='ACER')
process = measure_figure.Epc(ctx, scores, True, load.split, hter="ACER")
process.run()
@common_options.hist_command(
common_options.HIST_HELP.format(
score_format=SCORE_FORMAT, command='bob pad hist'))
common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob pad hist")
)
def hist(ctx, scores, evaluation, **kwargs):
process = figure.Hist(ctx, scores, evaluation, load.split)
process.run()
......@@ -79,21 +199,43 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.evaluate_command(
common_options.EVALUATE_HELP.format(
score_format=SCORE_FORMAT, command='bob pad evaluate'),
criteria=CRITERIA)
score_format=SCORE_FORMAT, command="bob pad evaluate"
),
criteria=CRITERIA,
)
def evaluate(ctx, scores, evaluation, **kwargs):
common_options.evaluate_flow(
ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs)
ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
)
@common_options.multi_metrics_command(
common_options.MULTI_METRICS_HELP.format(
names='FtA, APCER, BPCER, FAR, FRR, ACER',
criteria=CRITERIA, score_format=SCORE_FORMAT,
command='bob pad multi-metrics'),
criteria=CRITERIA)
def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)
process = figure.MultiMetrics(
ctx, scores, evaluation, load.split)
names="FtA, APCER, BPCER, FAR, FRR, ACER, HTER, precision, recall, f1_score",
criteria=CRITERIA,
score_format=SCORE_FORMAT,
command="bob pad multi-metrics",
),
criteria=CRITERIA,
epilog="""\b
More examples:
\b
bob pad multi-metrics -vvv -e -pn 6 -lg IQM,LBP -r print -r video \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_3_{1,2,3,4,5,6}/scores/scores-{dev,eval}
See also ``bob pad metrics``.
""",
)
@regexps_option()
@regexp_column_option()
@metrics_option(default="fta,apcer_pais,apcer,bpcer,acer,hter")
def multi_metrics(
ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs
):
ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
load_fn = partial(
negatives_per_pai_and_positives, regexps=regexps, regexp_column=regexp_column
)
process = figure.MultiMetrics(ctx, scores, evaluation, load_fn, metrics)
process.run()
This diff is collapsed.
This diff is collapsed.
from bob.io.base.test_utils import datafile
from bob.io.base import HDF5File
from bob.pad.base.script.error_utils import (
negatives_per_pai_and_positives,
apcer_bpcer,
calc_threshold,
)
import nose
import numpy as np
GENERATE_REFERENCES = False
scores_dev = datafile("per_pai_scores/scores-dev", module=__name__)
scores_dev_reference_mask = datafile(
"per_pai_scores/scores-dev-{i}.hdf5", module=__name__
)
def _dump_dict(f, d, name):
f[f"{name}_len"] = len(d)
for i, (k, v) in enumerate(d.items()):
f[f"{name}_key_{i}"] = k
f[f"{name}_value_{i}"] = v
def _read_dict(f, name):
ret = dict()
for i in range(f[f"{name}_len"]):
k = f[f"{name}_key_{i}"]
v = f[f"{name}_value_{i}"]
if isinstance(v, np.ndarray):
v = v.tolist()
ret[k] = v
return ret
def test_per_pai_apcer():
for i, regexps in enumerate((None, ["x[0-2]", "x[3-4]"], ["x[1-2]", "x[3-4]"])):
try:
pos, negs = negatives_per_pai_and_positives(scores_dev, regexps)
except ValueError:
if i == 2:
continue
raise
all_negs = [s for scores in negs.values() for s in scores]
thresholds = dict()
for method in ("bpcer20", "far", "eer", "min-hter"):
thresholds[method] = calc_threshold(
method, pos, negs.values(), all_negs, far_value=0.1
)
metrics = dict()
for method, threshold in thresholds.items():
apcers, apcer, bpcer = apcer_bpcer(threshold, pos, *negs.values())
metrics[method] = apcers + [apcer, bpcer]
scores_dev_reference = scores_dev_reference_mask.format(i=i)
if GENERATE_REFERENCES:
with HDF5File(scores_dev_reference, "w") as f:
f["pos"] = pos
_dump_dict(f, negs, "negs")
_dump_dict(f, thresholds, "thresholds")
_dump_dict(f, metrics, "metrics")
with HDF5File(scores_dev_reference, "r") as f:
ref_pos = f["pos"].tolist()
ref_negs = _read_dict(f, "negs")
ref_thresholds = _read_dict(f, "thresholds")
ref_metrics = _read_dict(f, "metrics")
nose.tools.assert_list_equal(pos, ref_pos)
nose.tools.assert_dict_equal(negs, ref_negs)
nose.tools.assert_dict_equal(thresholds, ref_thresholds)
nose.tools.assert_dict_equal(metrics, ref_metrics)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment