Commit cd6a04d2 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Add histograms, vulnerability, epc, epsc, gen commands and corresponding tests

parent 01f51991
Pipeline #18941 failed with stage
in 39 minutes and 55 seconds
"""Generates PAD ISO compliant EPC based on the score files
"""
import click
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
@click.command()
@common_options.scores_argument(eval_mandatory=True, min_len=2, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.titles_option()
@common_options.axis_fontsize_option()
@common_options.const_layout_option()
@common_options.figsize_option()
@common_options.bool_option(
'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
"""Plot EPC (expected performance curve):
You need to provide 4 score
files for each biometric system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
vulnerability analysis.
Examples:
$ bob pad epc dev-scores eval-scores
$ bob pad epc -o my_epc.pdf dev-scores1 eval-scores1
$ bob pad epc {licit,spoof}/scores-{dev,eval}
"""
process = figure.Epc(ctx, scores, True, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(eval_mandatory=True, min_len=2, nargs=-1)
@common_options.output_plot_file_option(default_out='epsc.pdf')
@common_options.titles_option()
@common_options.figsize_option()
@common_options.axis_fontsize_option()
@common_options.const_layout_option()
@common_options.bool_option(
'wer', 'w', 'Whether to plot the WER related lines or not.', True
)
@common_options.bool_option(
'three-d', 'D', 'If true, generate 3D plots', False
)
@common_options.bool_option(
'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', False
)
@click.option('-c', '--criteria', default="eer", show_default=True,
help='Criteria for threshold selection',
type=click.Choice(('eer', 'hter', 'wer')))
@click.option('-vp', '--var-param', default="omega", show_default=True,
help='Name of the varying parameter',
type=click.Choice(('omega', 'beta')))
@click.option('-fp', '--fixed-param', default=0.5, show_default=True,
help='Value of the fixed parameter',
type=click.FLOAT)
@verbosity_option()
@click.pass_context
def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, **kwargs):
"""Plot EPSC (expected performance curve):
You need to provide 4 score
files for each biometric system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
vulnerability analysis.
Note that when using 3D plots with option ``--three-d``, you cannot plot
both WER and IAPMR on the same figure (which is possible in 2D).
Examples:
$ bob pad epsc dev-scores eval-scores
$ bob pad epsc -o my_epsc.pdf dev-scores1 eval-scores1
$ bob pad epsc -D {licit,spoof}/scores-{dev,eval}
"""
if three_d:
if (ctx.meta['wer'] and ctx.meta['iapmr']):
raise click.BadParameter('Cannot plot both WER and IAPMR in 3D')
process = figure.Epsc3D(
ctx, scores, True, FUNC_SPLIT,
criteria, var_param, fixed_param
)
else:
process = figure.Epsc(
ctx, scores, True, FUNC_SPLIT,
criteria, var_param, fixed_param
)
process.run()
This diff is collapsed.
This diff is collapsed.
"""Generate random scores.
"""
import pkg_resources # to make sure bob gets imported properly
import os
import logging
import numpy
import click
from click.types import FLOAT
from bob.extension.scripts.click_helper import verbosity_option
from bob.core import random
from bob.io.base import create_directories_safe
logger = logging.getLogger(__name__)
NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000
def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
sigma_pa=1):
mt = random.mt19937() # initialise the random number generator
genuine_generator = random.normal(numpy.float32, mean_gen, sigma_gen)
zei_generator = random.normal(numpy.float32, mean_zei, sigma_zei)
pa_generator = random.normal(numpy.float32, mean_pa, sigma_pa)
genuine_scores = [genuine_generator(mt) for i in range(NUM_GENUINE_ACCESS)]
zei_scores = [zei_generator(mt) for i in range(NUM_ZEIMPOSTORS)]
pa_scores = [pa_generator(mt) for i in range(NUM_PA)]
return genuine_scores, zei_scores, pa_scores
def write_scores_to_file(pos, neg, filename, attack=False):
"""Writes score distributions into 4-column score files. For the format of
the 4-column score files, please refer to Bob's documentation.
Parameters
----------
pos : array_like
Scores for positive samples.
neg : array_like
Scores for negative samples.
filename : str
The path to write the score to.
"""
create_directories_safe(os.path.dirname(filename))
with open(filename, 'wt') as f:
for i in pos:
f.write('x x foo %f\n' % i)
for i in neg:
if attack:
f.write('x attack foo %f\n' % i)
else:
f.write('x y foo %f\n' % i)
@click.command()
@click.argument('outdir')
@click.option('--mean-gen', default=10, type=FLOAT, show_default=True)
@click.option('--mean-zei', default=0, type=FLOAT, show_default=True)
@click.option('--mean-pa', default=5, type=FLOAT, show_default=True)
@verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa):
"""Generate random scores.
Generates random scores for three types of verification attempts:
genuine users, zero-effort impostors and spoofing attacks and writes them
into 4-column score files for so called licit and spoof scenario. The
scores are generated using Gaussian distribution whose mean is an input
parameter. The generated scores can be used as hypothetical datasets.
"""
# Generate the data
genuine_dev, zei_dev, pa_dev = gen_score_distr(
mean_gen, mean_zei, mean_pa)
genuine_eval, zei_eval, pa_eval = gen_score_distr(
mean_gen, mean_zei, mean_pa)
# Write the data into files
write_scores_to_file(genuine_dev, zei_dev,
os.path.join(outdir, 'licit', 'scores-dev'))
write_scores_to_file(genuine_eval, zei_eval,
os.path.join(outdir, 'licit', 'scores-eval'))
write_scores_to_file(genuine_dev, pa_dev,
os.path.join(outdir, 'spoof', 'scores-dev'),
attack=True)
write_scores_to_file(genuine_eval, pa_eval,
os.path.join(outdir, 'spoof', 'scores-eval'),
attack=True)
"""Generates PAD ISO compliant histograms based on the score files
"""
import click
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option(dflt=False)
@common_options.titles_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
""" Plots histograms of Bona fida and PA along with threshold
criterion.
You need provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
computed from dev-scores. If you want to display dev-scores distributions
as well, use ``--show-dev`` option.
Examples:
$ bob pad hist dev-scores
$ bob pad hist dev-scores1 eval-scores1 dev-scores2
eval-scores2
$ bob pad hist --criter hter dev-scores1 eval-scores1
"""
process = figure.HistPad(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1, eval_mandatory=True, min_len=2)
@common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.axis_fontsize_option()
@common_options.thresholds_option()
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option(dflt=False)
@common_options.bool_option(
'iapmr-line', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@common_options.bool_option(
'real-data', 'R',
'If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.', True
)
@common_options.titles_option()
@verbosity_option()
@click.pass_context
def vuln(ctx, scores, evaluation, **kwargs):
'''Vulnerability analysis distributions.
Plots the histogram of score distributions. You need to provide 4 score
files for each biometric system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
vulnerability analysis.
You need provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used set the flag `--no-evaluation`
is required in that case.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
computed from dev-scores. If you want to display dev-scores distributions
as well, use ``--show-dev`` option.
Examples:
$ bob pad vuln licit/scores-dev licit/scores-eval \
spoof/scores-dev spoof/scores-eval
$ bob pad vuln {licit,spoof}/scores-{dev,eval}
'''
process = figure.HistVuln(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
"""Calculates PAD ISO compliant metrics based on the score files
"""
import logging
import click
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
from bob.measure.load import split
from bob.measure import (
farfrr, far_threshold, eer_threshold, min_hter_threshold)
logger = logging.getLogger(__name__)
ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')
def scores_dev_eval(development_scores, evaluation_scores):
dev_neg, dev_pos = split(development_scores)
dev_neg.sort()
dev_pos.sort()
if evaluation_scores is None:
logger.debug("No evaluation scores were provided.")
eval_neg, eval_pos = None, None
else:
eval_neg, eval_pos = split(evaluation_scores)
eval_neg.sort()
eval_pos.sort()
return dev_neg, dev_pos, eval_neg, eval_pos
def report(dev_neg, dev_pos, eval_neg, eval_pos, threshold):
for group, neg, pos in [
('Development', dev_neg, dev_pos),
('Evaluation', eval_neg, eval_pos),
]:
if neg is None:
continue
click.echo("{} set:".format(group))
apcer, bpcer = farfrr(neg, pos, threshold)
click.echo("APCER: {:>5.1f}%".format(apcer * 100))
click.echo("BPCER: {:>5.1f}%".format(bpcer * 100))
click.echo("HTER: {:>5.1f}%".format((apcer + bpcer) * 50))
from bob.bio.base.score import load
from . import figure
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@click.argument('development_scores')
@click.argument('evaluation_scores', required=False)
@click.option(
'-c', '--criterion', multiple=True, default=['bpcer20'],
type=click.Choice(ALL_CRITERIA), help='The criteria to select. You can '
'select multiple criteria by passing this option multiple times.',
show_default=True)
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
@common_options.table_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.titles_option()
@verbosity_option()
def metrics(development_scores, evaluation_scores, criterion):
@click.pass_context
def metrics(ctx, scores, evaluation, **kwargs):
"""PAD ISO compliant metrics.
Reports several metrics based on a selected threshold on the development
set. The thresholds are selected based on different criteria:
Reports several metrics based on a selected thresholds on the development
set and apply them on evaluation sets (if provided). The used thresholds are:
bpcer20 When APCER is set to 5%.
......@@ -60,6 +29,9 @@ def metrics(development_scores, evaluation_scores, criterion):
min-hter When HTER is minimum.
This command produces one table per sytem. Format of the table can be
changed through option ``--tablefmt``.
Most metrics are according to the ISO/IEC 30107-3:2017 "Information
technology -- Biometric presentation attack detection -- Part 3: Testing
and reporting" standard. The reported metrics are:
......@@ -76,21 +48,7 @@ def metrics(development_scores, evaluation_scores, criterion):
$ bob pad metrics /path/to/scores-dev /path/to/scores-eval
$ bob pad metrics /path/to/scores-{dev,eval} # using bash expansion
$ bob pad metrics -c bpcer20 -c eer /path/to/scores-dev
$ bob pad metrics /path/to/system{1,2,3}/score-{dev,eval}
"""
dev_neg, dev_pos, eval_neg, eval_pos = scores_dev_eval(
development_scores, evaluation_scores)
for method in criterion:
if method == 'bpcer20':
threshold = far_threshold(dev_neg, dev_pos, 0.05, True)
elif method == 'eer':
threshold = eer_threshold(dev_neg, dev_pos, True)
elif method == 'min-hter':
threshold = min_hter_threshold(dev_neg, dev_pos, True)
else:
raise ValueError("Unknown threshold criteria: {}".format(method))
click.echo("\nThreshold of {} selected with the {} criteria".format(
threshold, method))
report(dev_neg, dev_pos, eval_neg, eval_pos, threshold)
process = figure.Metrics(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
import sys
import click
from click.testing import CliRunner
import pkg_resources
from ..script import (metrics, histograms, epc)
def test_hist():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(histograms.hist, ['--no-evaluation', licit_dev])
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(histograms.hist, ['--criter', 'hter', '--output',
'HISTO.pdf', '-b',
30, '--no-evaluation',
licit_dev, spoof_dev])
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(histograms.hist, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F',
3, licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
def test_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(histograms.vuln, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F',
3, licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
def test_epc():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(epc.epc, ['--output', 'epc.pdf',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
result = runner.invoke(epc.epc, ['--output', 'epc.pdf', '-I',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
def test_epsc():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(epc.epsc, ['--output', 'epsc.pdf',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
result = runner.invoke(epc.epsc, ['--output', 'epsc.pdf', '-I',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
result = runner.invoke(epc.epsc, ['--output', 'epsc.pdf', '-D',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
result = runner.invoke(epc.epsc, ['--output', 'epsc.pdf', '-D',
'-I', '--no-wer',
licit_dev, licit_test,
spoof_dev, spoof_test])
assert result.exit_code == 0
......@@ -140,6 +140,11 @@ setup(
# bob pad scripts
'bob.pad.cli': [
'metrics = bob.pad.base.script.metrics:metrics',
'hist = bob.pad.base.script.histograms:hist',
'vuln = bob.pad.base.script.histograms:vuln',
'epc = bob.pad.base.script.epc:epc',
'epsc = bob.pad.base.script.epc:epsc',
'gen = bob.pad.base.script.gen:gen',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment