Commit 0b85f0c6 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'finalplot' into 'master'

Various fix

See merge request !48
parents dd69e7f3 3defd9c3
Pipeline #21699 passed with stages
in 15 minutes and 42 seconds
...@@ -37,6 +37,8 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys): ...@@ -37,6 +37,8 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys):
@common_options.metrics_command(common_options.METRICS_HELP.format( @common_options.metrics_command(common_options.METRICS_HELP.format(
names='FtA, APCER, BPCER, FAR, FRR, ACER', names='FtA, APCER, BPCER, FAR, FRR, ACER',
criteria=CRITERIA, score_format=SCORE_FORMAT, criteria=CRITERIA, score_format=SCORE_FORMAT,
hter_note='Note that FAR = APCER * (1 - FtA), '
'FRR = FtA + BPCER * (1 - FtA) and ACER = (APCER + BPCER) / 2.',
command='bob pad metrics'), criteria=CRITERIA) command='bob pad metrics'), criteria=CRITERIA)
def metrics(ctx, scores, evaluation, **kwargs): def metrics(ctx, scores, evaluation, **kwargs):
process = figure.Metrics(ctx, scores, evaluation, load.split) process = figure.Metrics(ctx, scores, evaluation, load.split)
......
...@@ -7,13 +7,14 @@ from .error_utils import calc_threshold ...@@ -7,13 +7,14 @@ from .error_utils import calc_threshold
ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter') ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')
class Metrics(measure_figure.Metrics): class Metrics(bio_figure.Metrics):
'''Compute metrics from score files''' '''Compute metrics from score files'''
def __init__(self, ctx, scores, evaluation, func_load): def __init__(self, ctx, scores, evaluation, func_load,
names=('FtA', 'APCER', 'BPCER', 'FAR', 'FRR', 'HTER')):
super(Metrics, self).__init__( super(Metrics, self).__init__(
ctx, scores, evaluation, func_load, ctx, scores, evaluation, func_load, names
names=('FtA', 'APCER', 'BPCER', 'FAR', 'FRR', 'ACER')) )
def get_thres(self, criterion, dev_neg, dev_pos, far): def get_thres(self, criterion, dev_neg, dev_pos, far):
if self._criterion == 'bpcer20': if self._criterion == 'bpcer20':
...@@ -61,7 +62,7 @@ class Hist(measure_figure.Hist): ...@@ -61,7 +62,7 @@ class Hist(measure_figure.Hist):
def _setup_hist(self, neg, pos): def _setup_hist(self, neg, pos):
self._title_base = 'PAD' self._title_base = 'PAD'
self._density_hist( self._density_hist(
pos[0], n=0, label='Bona Fide', color='C1' pos[0], n=0, label='Bona-fide', color='C1'
) )
self._density_hist( self._density_hist(
neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7', neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
......
...@@ -2,17 +2,20 @@ ...@@ -2,17 +2,20 @@
""" """
import os import os
import logging
import numpy import numpy
import click import click
from click.types import FLOAT from click.types import FLOAT
from bob.measure.script import common_options from bob.measure.script import common_options
from bob.extension.scripts.click_helper import ( from bob.extension.scripts.click_helper import (
verbosity_option, bool_option, list_float_option) verbosity_option, bool_option, list_float_option
)
from bob.core import random from bob.core import random
from bob.io.base import create_directories_safe from bob.io.base import create_directories_safe
from bob.bio.base.score import load from bob.bio.base.score import load
from . import vuln_figure as figure from . import vuln_figure as figure
LOGGER = logging.getLogger(__name__)
NUM_GENUINE_ACCESS = 5000 NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000 NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000 NUM_PA = 5000
...@@ -70,9 +73,9 @@ def write_scores_to_file(neg, pos, filename, attack=False): ...@@ -70,9 +73,9 @@ def write_scores_to_file(neg, pos, filename, attack=False):
@click.command() @click.command()
@click.argument('outdir') @click.argument('outdir')
@click.option('--mean-gen', default=7, type=FLOAT, show_default=True) @click.option('-mg', '--mean-gen', default=7, type=FLOAT, show_default=True)
@click.option('--mean-zei', default=3, type=FLOAT, show_default=True) @click.option('-mz', '--mean-zei', default=3, type=FLOAT, show_default=True)
@click.option('--mean-pa', default=5, type=FLOAT, show_default=True) @click.option('-mp', '--mean-pa', default=5, type=FLOAT, show_default=True)
@verbosity_option() @verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa): def gen(outdir, mean_gen, mean_zei, mean_pa):
"""Generate random scores. """Generate random scores.
...@@ -103,7 +106,7 @@ def gen(outdir, mean_gen, mean_zei, mean_pa): ...@@ -103,7 +106,7 @@ def gen(outdir, mean_gen, mean_zei, mean_pa):
@click.command() @click.command()
@common_options.scores_argument(min_arg=2, nargs=-1) @common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_roc.pdf') @common_options.output_plot_file_option(default_out='roc.pdf')
@common_options.legends_option() @common_options.legends_option()
@common_options.no_legend_option() @common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right') @common_options.legend_loc_option(dflt='upper-right')
...@@ -143,7 +146,7 @@ def roc(ctx, scores, real_data, **kwargs): ...@@ -143,7 +146,7 @@ def roc(ctx, scores, real_data, **kwargs):
@click.command() @click.command()
@common_options.scores_argument(min_arg=2, nargs=-1) @common_options.scores_argument(min_arg=2, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_det.pdf') @common_options.output_plot_file_option(default_out='det.pdf')
@common_options.legends_option() @common_options.legends_option()
@common_options.no_legend_option() @common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right') @common_options.legend_loc_option(dflt='upper-right')
...@@ -183,7 +186,7 @@ def det(ctx, scores, real_data, **kwargs): ...@@ -183,7 +186,7 @@ def det(ctx, scores, real_data, **kwargs):
@click.command() @click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1) @common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epc.pdf') @common_options.output_plot_file_option(default_out='epc.pdf')
@common_options.legends_option() @common_options.legends_option()
@common_options.no_legend_option() @common_options.no_legend_option()
@common_options.legend_loc_option() @common_options.legend_loc_option()
...@@ -227,23 +230,25 @@ def epc(ctx, scores, **kwargs): ...@@ -227,23 +230,25 @@ def epc(ctx, scores, **kwargs):
@click.command() @click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1) @common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epsc.pdf') @common_options.output_plot_file_option(default_out='epsc.pdf')
@common_options.titles_option()
@common_options.legends_option() @common_options.legends_option()
@common_options.no_legend_option() @common_options.no_legend_option()
@common_options.legend_loc_option() @common_options.legend_ncols_option()
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.x_label_option() @common_options.x_label_option()
@common_options.y_label_option() @common_options.y_label_option()
@common_options.figsize_option(dflt=None) @common_options.figsize_option(dflt='5,3')
@common_options.style_option() @common_options.style_option()
@common_options.bool_option( @common_options.bool_option(
'wer', 'w', 'Whether to plot the WER related lines or not.', True 'wer', 'w', 'Whether to plot the WER related lines or not.', True
) )
@common_options.bool_option( @common_options.bool_option(
'three-d', 'D', 'If true, generate 3D plots', False 'three-d', 'D', 'If true, generate 3D plots. You need to turn off '
'wer or iapmr when using this option.', False
) )
@common_options.bool_option( @common_options.bool_option(
'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', False 'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
) )
@click.option('-c', '--criteria', default="eer", show_default=True, @click.option('-c', '--criteria', default="eer", show_default=True,
help='Criteria for threshold selection', help='Criteria for threshold selection',
...@@ -251,14 +256,13 @@ def epc(ctx, scores, **kwargs): ...@@ -251,14 +256,13 @@ def epc(ctx, scores, **kwargs):
@click.option('-vp', '--var-param', default="omega", show_default=True, @click.option('-vp', '--var-param', default="omega", show_default=True,
help='Name of the varying parameter', help='Name of the varying parameter',
type=click.Choice(('omega', 'beta'))) type=click.Choice(('omega', 'beta')))
@click.option('-fp', '--fixed-param', default=0.5, show_default=True, @list_float_option(name='fixed-params', short_name='fp', dflt='0.5',
help='Value of the fixed parameter', desc='Values of the fixed parameter, separated by commas')
type=click.FLOAT)
@click.option('-s', '--sampling', default=5, show_default=True, @click.option('-s', '--sampling', default=5, show_default=True,
help='Sampling of the EPSC 3D surface', type=click.INT) help='Sampling of the EPSC 3D surface', type=click.INT)
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling, def epsc(ctx, scores, criteria, var_param, three_d, sampling,
**kwargs): **kwargs):
"""Plot EPSC (expected performance spoofing curve): """Plot EPSC (expected performance spoofing curve):
...@@ -282,25 +286,27 @@ def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling, ...@@ -282,25 +286,27 @@ def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling,
$ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval} $ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval}
""" """
fixed_params = ctx.meta.get('fixed_params', [0.5])
if three_d: if three_d:
if (ctx.meta['wer'] and ctx.meta['iapmr']): if (ctx.meta['wer'] and ctx.meta['iapmr']):
raise click.BadParameter('Cannot plot both WER and IAPMR in 3D') LOGGER.info('Cannot plot both WER and IAPMR in 3D. Will turn IAPMR off.')
ctx.meta['iapmr'] = False
ctx.meta['sampling'] = sampling ctx.meta['sampling'] = sampling
process = figure.Epsc3D( process = figure.Epsc3D(
ctx, scores, True, load.split, ctx, scores, True, load.split,
criteria, var_param, fixed_param criteria, var_param, fixed_params
) )
else: else:
process = figure.Epsc( process = figure.Epsc(
ctx, scores, True, load.split, ctx, scores, True, load.split,
criteria, var_param, fixed_param criteria, var_param, fixed_params
) )
process.run() process.run()
@click.command() @click.command()
@common_options.scores_argument(nargs=-1, min_arg=2) @common_options.scores_argument(nargs=-1, min_arg=2)
@common_options.output_plot_file_option(default_out='vuln_hist.pdf') @common_options.output_plot_file_option(default_out='hist.pdf')
@common_options.n_bins_option() @common_options.n_bins_option()
@common_options.criterion_option() @common_options.criterion_option()
@common_options.thresholds_option() @common_options.thresholds_option()
...@@ -346,10 +352,12 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -346,10 +352,12 @@ def hist(ctx, scores, evaluation, **kwargs):
Examples: Examples:
$ bob vuln vuln_hist -e -v licit/scores-dev licit/scores-eval \ $ bob vuln hist -v licit/scores-dev spoof/scores-dev
$ bob vuln hist -e -v licit/scores-dev licit/scores-eval \
spoof/scores-dev spoof/scores-eval spoof/scores-dev spoof/scores-eval
$ bob vuln vuln_hist -e -v {licit,spoof}/scores-{dev,eval} $ bob vuln hist -e -v {licit,spoof}/scores-{dev,eval}
''' '''
process = figure.HistVuln(ctx, scores, evaluation, load.split) process = figure.HistVuln(ctx, scores, evaluation, load.split)
process.run() process.run()
...@@ -364,7 +372,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -364,7 +372,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.title_option() @common_options.title_option()
@common_options.const_layout_option() @common_options.const_layout_option()
@common_options.style_option() @common_options.style_option()
@common_options.figsize_option(dflt=None) @common_options.figsize_option()
@verbosity_option() @verbosity_option()
@common_options.axes_val_option() @common_options.axes_val_option()
@common_options.x_rotation_option() @common_options.x_rotation_option()
......
This diff is collapsed.
...@@ -8,6 +8,6 @@ from bob.extension.scripts.click_helper import AliasedGroup ...@@ -8,6 +8,6 @@ from bob.extension.scripts.click_helper import AliasedGroup
@with_plugins(pkg_resources.iter_entry_points('bob.vuln.cli')) @with_plugins(pkg_resources.iter_entry_points('bob.vuln.cli'))
@click.group(cls=AliasedGroup) @click.group(cls=AliasedGroup)
def vuln(): def vulnerability():
"""Vulnerability analysis related commands.""" """Vulnerability analysis related commands."""
pass pass
...@@ -13,7 +13,7 @@ def test_det_pad(): ...@@ -13,7 +13,7 @@ def test_det_pad():
result = runner.invoke(pad_commands.det, ['-e', '--output', result = runner.invoke(pad_commands.det, ['-e', '--output',
'DET.pdf', 'DET.pdf',
licit_dev, licit_test]) licit_dev, licit_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_det_vuln(): def test_det_vuln():
...@@ -32,7 +32,7 @@ def test_det_vuln(): ...@@ -32,7 +32,7 @@ def test_det_vuln():
'DET.pdf', 'DET.pdf',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_fmr_iapmr_vuln(): def test_fmr_iapmr_vuln():
...@@ -50,13 +50,13 @@ def test_fmr_iapmr_vuln(): ...@@ -50,13 +50,13 @@ def test_fmr_iapmr_vuln():
'--output', 'FMRIAPMR.pdf', licit_dev, licit_test, spoof_dev, '--output', 'FMRIAPMR.pdf', licit_dev, licit_test, spoof_dev,
spoof_test spoof_test
]) ])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
result = runner.invoke(vuln_commands.fmr_iapmr, [ result = runner.invoke(vuln_commands.fmr_iapmr, [
'--output', 'FMRIAPMR.pdf', licit_dev, licit_test, spoof_dev, '--output', 'FMRIAPMR.pdf', licit_dev, licit_test, spoof_dev,
spoof_test, '-G', '-L', '1e-7,1,0,1' spoof_test, '-G', '-L', '1e-7,1,0,1'
]) ])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_hist_pad(): def test_hist_pad():
...@@ -71,7 +71,7 @@ def test_hist_pad(): ...@@ -71,7 +71,7 @@ def test_hist_pad():
runner = CliRunner() runner = CliRunner()
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke(pad_commands.hist, [licit_dev]) result = runner.invoke(pad_commands.hist, [licit_dev])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke(pad_commands.hist, ['--criterion', 'min-hter', result = runner.invoke(pad_commands.hist, ['--criterion', 'min-hter',
...@@ -79,7 +79,7 @@ def test_hist_pad(): ...@@ -79,7 +79,7 @@ def test_hist_pad():
'HISTO.pdf', '-b', 'HISTO.pdf', '-b',
'30,20', '30,20',
licit_dev, spoof_dev]) licit_dev, spoof_dev])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke(pad_commands.hist, ['-e', '--criterion', 'eer', result = runner.invoke(pad_commands.hist, ['-e', '--criterion', 'eer',
...@@ -87,7 +87,7 @@ def test_hist_pad(): ...@@ -87,7 +87,7 @@ def test_hist_pad():
'HISTO.pdf', '-b', '30', 'HISTO.pdf', '-b', '30',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_hist_vuln(): def test_hist_vuln():
...@@ -105,7 +105,7 @@ def test_hist_vuln(): ...@@ -105,7 +105,7 @@ def test_hist_vuln():
['--criterion', 'eer', '--output', ['--criterion', 'eer', '--output',
'HISTO.pdf', '-b', '30', '-ts', 'A,B', 'HISTO.pdf', '-b', '30', '-ts', 'A,B',
licit_dev, licit_test]) licit_dev, licit_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke(vuln_commands.hist, result = runner.invoke(vuln_commands.hist,
...@@ -113,7 +113,7 @@ def test_hist_vuln(): ...@@ -113,7 +113,7 @@ def test_hist_vuln():
'HISTO.pdf', '-b', '2,20,30', '-e', 'HISTO.pdf', '-b', '2,20,30', '-e',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_metrics_pad(): def test_metrics_pad():
...@@ -127,7 +127,7 @@ def test_metrics_pad(): ...@@ -127,7 +127,7 @@ def test_metrics_pad():
pad_commands.metrics, pad_commands.metrics,
['-e', licit_dev, licit_test] ['-e', licit_dev, licit_test]
) )
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_epc_vuln(): def test_epc_vuln():
...@@ -145,13 +145,13 @@ def test_epc_vuln(): ...@@ -145,13 +145,13 @@ def test_epc_vuln():
['--output', 'epc.pdf', ['--output', 'epc.pdf',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
result = runner.invoke(vuln_commands.epc, result = runner.invoke(vuln_commands.epc,
['--output', 'epc.pdf', '-I', ['--output', 'epc.pdf', '-I',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_epsc_vuln(): def test_epsc_vuln():
...@@ -173,22 +173,34 @@ def test_epsc_vuln(): ...@@ -173,22 +173,34 @@ def test_epsc_vuln():
result = runner.invoke(vuln_commands.epsc, result = runner.invoke(vuln_commands.epsc,
['--output', 'epsc.pdf', '-I', ['--output', 'epsc.pdf', '-I',
'-fp', '0.1,0.3',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_epsc_3D_vuln():
licit_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-dev')
licit_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/licit/scores-eval')
spoof_dev = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-dev')
spoof_test = pkg_resources.resource_filename('bob.pad.base.test',
'data/spoof/scores-eval')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(vuln_commands.epsc, result = runner.invoke(vuln_commands.epsc,
['--output', 'epsc.pdf', '-D', ['--output', 'epsc.pdf', '-D',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
result = runner.invoke(vuln_commands.epsc, result = runner.invoke(vuln_commands.epsc,
['--output', 'epsc.pdf', '-D', ['--output', 'epsc.pdf', '-D',
'-I', '--no-wer', '-I', '--no-wer',
licit_dev, licit_test, licit_dev, licit_test,
spoof_dev, spoof_test]) spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
def test_evaluate_pad(): def test_evaluate_pad():
...@@ -204,4 +216,4 @@ def test_evaluate_pad(): ...@@ -204,4 +216,4 @@ def test_evaluate_pad():
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke(pad_commands.evaluate, result = runner.invoke(pad_commands.evaluate,
[licit_dev, licit_test, spoof_dev, spoof_test]) [licit_dev, licit_test, spoof_dev, spoof_test])
assert result.exit_code == 0, (result.exit_code, result.output) assert result.exit_code == 0, (result.exit_code, result.output, result.exception)
...@@ -134,7 +134,7 @@ setup( ...@@ -134,7 +134,7 @@ setup(
# main entry for bob pad cli # main entry for bob pad cli
'bob.cli': [ 'bob.cli': [
'pad = bob.pad.base.script.pad:pad', 'pad = bob.pad.base.script.pad:pad',
'vuln = bob.pad.base.script.vuln:vuln', 'vulnerability = bob.pad.base.script.vulnerability:vulnerability',
], ],
# bob pad scripts # bob pad scripts
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment