Skip to content
GitLab
Commit b63237ce authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

nit

parent 2ddbe247
Pipeline #19959 canceled with stage
in 6 minutes and 28 seconds
Hide whitespace changes
Inline Side-by-side
bob/pad/base/script/det.py
View file @ b63237ce
......@@ -6,6 +6,7 @@ from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
......@@ -31,23 +32,23 @@ from . import figure
help='If False, will annotate the plots hypothetically, instead '
'of with real data values of the calculated error rates.')
@click.pass_context
def det(ctx, scores, criteria, real_data, **kwargs):
"""Plot DET
def det(ctx, scores, criteria, real_data, **kwargs):
"""Plot DET
You need to provide 2 or 4 scores
files for each PAD system in this order:
You need to provide 2 or 4 scores
files for each PAD system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores (when ``--no-spoof`` is False (default))
* spoof evaluation scores (when ``--no-spoof`` is False (default))
\b
* licit development scores
* licit evaluation scores
* spoof development scores (when ``--no-spoof`` is False (default))
* spoof evaluation scores (when ``--no-spoof`` is False (default))
Examples:
$ bob pad det --no-spoof dev-scores eval-scores
Examples:
$ bob pad det --no-spoof dev-scores eval-scores
$ bob pad det {licit,spoof}/scores-{dev,eval}
"""
process = figure.Det(ctx, scores, True, load.split, criteria, real_data)
process.run()
$ bob pad det {licit,spoof}/scores-{dev,eval}
"""
process = figure.Det(ctx, scores, True, load.split, criteria, real_data)
process.run()
bob/pad/base/script/epc.py
View file @ b63237ce
......@@ -6,6 +6,7 @@ from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
......@@ -49,6 +50,7 @@ def epc(ctx, scores, **kwargs):
process = figure.Epc(ctx, scores, True, load.split)
process.run()
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epsc.pdf')
......
bob/pad/base/script/error_utils.py
View file @ b63237ce
#!/usr/bin/env python
#Ivana Chingovska <ivana.chingovska@idiap.ch>
#Fri Dec 7 12:33:37 CET 2012
# Ivana Chingovska <ivana.chingovska@idiap.ch>
# Fri Dec 7 12:33:37 CET 2012
"""Utility functions for computation of EPSC curve and related measurement"""
import os
import sys
import bob.measure
import numpy
import argparse
def calc_pass_rate(threshold, attacks):
......@@ -69,7 +66,7 @@ def weighted_neg_error_rate_criteria(data,
if beta == 0.5:
return abs(far_w - frr)
else:
#return abs(far_w - frr)
# return abs(far_w - frr)
return abs((1 - beta) * frr - beta * far_w)
elif criteria == 'min-hter':
......@@ -137,7 +134,7 @@ def weighted_negatives_threshold(licit_neg,
beta=0.5,
criteria='eer'):
"""Calculates the threshold for achieving the given criteria between the
FAR_w and the FRR, given the single value for the weight parameter
FAR_w and the FRR, given the single value for the weight parameter
balancing between impostors and spoofing attacks and a single value for the
parameter beta balancing between the real accesses and the negatives
(impostors and spoofing attacks)
......@@ -205,14 +202,14 @@ def epsc_thresholds(licit_neg,
spoofing attacks. If None, it is going to span the full range [0,1].
Otherwise, can be set to a fixed value or a list of values.
- beta - the value of the parameter beta, balancing between real accesses
and all the negatives (zero-effort impostors and spoofing attacks). If
and all the negatives (zero-effort impostors and spoofing attacks). If
None, it is going to span the full range [0,1]. Otherwise, can be set to a
fixed value or a list of values.
"""
step_size = 1 / float(points)
if omega == None:
if omega is None:
omega = numpy.array([(i * step_size) for i in range(points + 1)])
elif not isinstance(omega, list) and not isinstance(
omega, tuple) and not isinstance(omega, numpy.ndarray):
......@@ -220,7 +217,7 @@ def epsc_thresholds(licit_neg,
else:
omega = numpy.array(omega)
if beta == None:
if beta is None:
beta = numpy.array([(i * step_size) for i in range(points + 1)])
elif not isinstance(beta, list) and not isinstance(
beta, tuple) and not isinstance(beta, numpy.ndarray):
......@@ -288,8 +285,8 @@ negative samples (impostors and spoofing attacks).
spoof_neg, spoof_pos,
threshold) # calculate test frr @ threshold (spoof scenario)
frr = farfrr_licit[
1] # we can take this value from farfrr_spoof as well, it doesn't matter
# we can take this value from farfrr_spoof as well, it doesn't matter
frr = farfrr_licit[1]
far = farfrr_licit[0]
sfar = farfrr_spoof[0]
......@@ -302,8 +299,8 @@ negative samples (impostors and spoofing attacks).
def epsc_error_rates(licit_neg, licit_pos, spoof_neg, spoof_pos, thresholds,
omega, beta):
"""Calculates several error rates: FAR_w and WER_wb for the given weights
(omega and beta) and thresholds (the thresholds need to be computed first
"""Calculates several error rates: FAR_w and WER_wb for the given weights
(omega and beta) and thresholds (the thresholds need to be computed first
using the method: epsc_thresholds() before passing to this method)
Parameters
......@@ -350,8 +347,8 @@ def epsc_error_rates(licit_neg, licit_pos, spoof_neg, spoof_pos, thresholds,
def all_error_rates(licit_neg, licit_pos, spoof_neg, spoof_pos, thresholds,
omega, beta):
"""Calculates several error rates: FAR_w and WER_wb for the given weights
(omega and beta) and thresholds (the thresholds need to be computed first
"""Calculates several error rates: FAR_w and WER_wb for the given weights
(omega and beta) and thresholds (the thresholds need to be computed first
using the method: epsc_thresholds() before passing to this method)
Parameters
......@@ -427,7 +424,7 @@ def calc_aue(licit_neg,
- h_bound - higher bound of integration
- points - number of points to calculate EPSC
- criteria - the decision threshold criteria ('eer', 'wer' or 'min-hter')
- var_param - name of the parameter which is varied on the abscissa
- var_param - name of the parameter which is varied on the abscissa
('omega' or 'beta')
"""
......
bob/pad/base/script/evaluate.py
View file @ b63237ce
......@@ -3,10 +3,10 @@ experiements
"""
import click
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option, bool_option)
from bob.bio.base.score import load
from bob.extension.scripts.click_helper import verbosity_option
import bob.bio.base.script.commands as bio_commands
from . import (figure, histograms, metrics, det, fmr_iapmr, epc)
from . import (histograms, metrics, det, fmr_iapmr, epc)
@click.command()
@common_options.scores_argument(nargs=-1)
......@@ -99,8 +99,8 @@ def vuln(ctx, scores, **kwargs):
\b
1. Computes bob pad vuln_metrics
2. Plots EPC, EPSC, vulnerability histograms, fmr vs IAPMR to a multi-page PDF
file
2. Plots EPC, EPSC, vulnerability histograms, fmr vs IAPMR to a multi-page
PDF file
You need to provide 4 score files for each biometric system in this order:
......
bob/pad/base/script/figure.py
View file @ b63237ce
'''Runs error analysis on score sets, outputs metrics and plots'''
import logging
import click
import numpy as np
import matplotlib.pyplot as mpl
import bob.measure.script.figure as measure_figure
import bob.measure.script.figure as measure_figure
from tabulate import tabulate
from bob.extension.scripts.click_helper import verbosity_option
from bob.measure.utils import (get_fta, get_fta_list, get_thres)
from bob.measure.utils import get_fta_list
from bob.measure import (
far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
)
......@@ -58,6 +56,7 @@ class Metrics(measure_figure.Metrics):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
neg_list, pos_list, _ = get_fta_list(input_scores)
......@@ -97,23 +96,24 @@ class MetricsVuln(measure_figure.Metrics):
super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
neg_list, pos_list, _ = get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
criter = self._criterion or 'eer'
threshold = calc_threshold(criter, dev_neg, dev_pos) \
if self._thres is None else self._thres[idx]
if self._thres is None else self._thres[idx]
far, frr = farfrr(neg_list[1], pos_list[1], threshold)
iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
title = self._legends[idx] if self._legends is not None else None
headers = ['' or title, '%s (threshold=%.2g)' %
(criter.upper(), threshold)]
rows = []
rows.append(['FMR (%)', '{:>5.1f}%'.format(100*far)])
rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr*100)])
rows.append(['HTER (%)', '{:>5.1f}%'.format(50*(far+frr))])
rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100*iapmr)])
rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr * 100)])
rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
click.echo(
tabulate(rows, headers, self._tablefmt),
file=self.log_file
......@@ -190,20 +190,21 @@ class HistVuln(measure_figure.Hist):
def _lines(self, threshold, label, neg, pos, idx, **kwargs):
if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
#plot vertical line
# plot vertical line
super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
#plot iapmr_line
# plot iapmr_line
iapmr, _ = farfrr(neg[1], pos[0], threshold)
ax2 = mpl.twinx()
# we never want grid lines on axis 2
ax2.grid(False)
real_data = True if 'real_data' not in self._ctx.meta else \
self._ctx.meta['real_data']
self._ctx.meta['real_data']
_iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
n = idx % self._step_print
col = n % self._ncols
rest_print = self.n_systems - int(idx / self._step_print) * self._step_print
rest_print = self.n_systems - \
int(idx / self._step_print) * self._step_print
if col == self._ncols - 1 or n == rest_print - 1:
ax2.set_ylabel("IAPMR (%)", color='C3')
ax2.tick_params(axis='y', colors='red')
......@@ -213,19 +214,20 @@ class HistVuln(measure_figure.Hist):
class PadPlot(measure_figure.PlotBase):
'''Base class for PAD plots'''
def __init__(self, ctx, scores, evaluation, func_load):
super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
mpl.rcParams['figure.constrained_layout.use'] = self._clayout
def end_process(self):
'''Close pdf '''
#do not want to close PDF when running evaluate
# do not want to close PDF when running evaluate
if 'PdfPages' in self._ctx.meta and \
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
self._pdf_page.close()
def _plot_legends(self):
#legends for all axes
# legends for all axes
lines = []
labels = []
for ax in mpl.gcf().get_axes():
......@@ -239,15 +241,16 @@ class PadPlot(measure_figure.PlotBase):
class Epc(PadPlot):
''' Handles the plotting of EPC '''
def __init__(self, ctx, scores, evaluation, func_load):
super(Epc, self).__init__(ctx, scores, evaluation, func_load)
self._iapmr = True if 'iapmr' not in self._ctx.meta else \
self._ctx.meta['iapmr']
self._ctx.meta['iapmr']
self._title = self._title or ('EPC and IAPMR' if self._iapmr else
'EPC')
self._x_label = self._x_label or r"Weight $\beta$"
self._y_label = self._y_label or "WER (%)"
self._eval = True #always eval data with EPC
self._eval = True # always eval data with EPC
self._split = False
self._nb_figs = 1
......@@ -306,7 +309,7 @@ class Epc(PadPlot):
title = self._legends[idx] if self._legends is not None else self._title
if title.replace(' ', ''):
mpl.title(title)
#legends for all axes
# legends for all axes
self._plot_legends()
mpl.xticks(rotation=self._x_rotation)
self._pdf_page.savefig(mpl.gcf())
......@@ -314,17 +317,18 @@ class Epc(PadPlot):
class Epsc(PadPlot):
''' Handles the plotting of EPSC '''
def __init__(self, ctx, scores, evaluation, func_load,
criteria, var_param, fixed_param):
super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
self._iapmr = False if 'iapmr' not in self._ctx.meta else \
self._ctx.meta['iapmr']
self._ctx.meta['iapmr']
self._wer = True if 'wer' not in self._ctx.meta else \
self._ctx.meta['wer']
self._ctx.meta['wer']
self._criteria = 'eer' if criteria is None else criteria
self._var_param = "omega" if var_param is None else var_param
self._fixed_param = 0.5 if fixed_param is None else fixed_param
self._eval = True #always eval data with EPC
self._eval = True # always eval data with EPC
self._split = False
self._nb_figs = 1
self._title = ''
......@@ -364,7 +368,7 @@ class Epsc(PadPlot):
spoof_dev_neg,
spoof_dev_pos,
points=points,
criteria= self._criteria,
criteria=self._criteria,
omega=self._fixed_param
)
......@@ -372,12 +376,12 @@ class Epsc(PadPlot):
licit_eval_neg, licit_eval_pos, spoof_eval_neg,
spoof_eval_pos, thrs, omega, beta
) # error rates are returned in a list in the
# following order: frr, far, IAPMR, far_w, wer_w
# following order: frr, far, IAPMR, far_w, wer_w
ax1 = mpl.subplot(
111
) # EPC like curves for FVAS fused scores for weighted error rates
# between the negatives (impostors and Presentation attacks)
# between the negatives (impostors and Presentation attacks)
if self._wer:
if self._var_param == 'omega':
mpl.plot(
......@@ -427,11 +431,11 @@ class Epsc(PadPlot):
if self._var_param == 'omega':
if title is not None and title.replace(' ', ''):
mpl.title(title or (r"EPSC with $\beta$ = %.2f" %\
mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
self._fixed_param))
else:
if title is not None and title.replace(' ', ''):
mpl.title(title or (r"EPSC with $\omega$ = %.2f" %\
mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
self._fixed_param))
mpl.grid()
......@@ -444,6 +448,7 @@ class Epsc(PadPlot):
class Epsc3D(Epsc):
''' 3D EPSC plots for PAD'''
def compute(self, idx, input_scores, input_names):
''' Implements plots'''
licit_dev_neg = input_scores[0][0]
......@@ -513,10 +518,11 @@ class Epsc3D(Epsc):
class Det(PadPlot):
'''DET for PAD'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._no_spoof = False if 'no_spoof' not in ctx.meta else\
ctx.meta['no_spoof']
ctx.meta['no_spoof']
self._criteria = criteria
self._real_data = True if real_data is None else real_data
......@@ -645,12 +651,12 @@ class Det(PadPlot):
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
close pdf is needed '''
#only for plots
# only for plots
add = ''
if not self._no_spoof:
add = " and overlaid SPOOF scenario"
title = self._title if self._title is not None else \
('DET: LICIT' + add)
('DET: LICIT' + add)
if title.replace(' ', ''):
mpl.title(title)
mpl.xlabel(self._x_label or "False Acceptance Rate (%)")
......@@ -669,9 +675,9 @@ class Det(PadPlot):
self._pdf_page.savefig(fig)
#do not want to close PDF when running evaluate
# do not want to close PDF when running evaluate
if 'PdfPages' in self._ctx.meta and \
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
self._pdf_page.close()
def _set_axis(self):
......@@ -683,13 +689,14 @@ class Det(PadPlot):
class FmrIapmr(PadPlot):
'''FMR vs IAPMR'''
def __init__(self, ctx, scores, evaluation, func_load):
super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
self._eval = True #always eval data with EPC
self._eval = True # always eval data with EPC
self._split = False
self._nb_figs = 1
self._semilogx = False if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
ctx.meta['semilogx']
if self._min_arg != 4:
raise click.BadParameter("You must provide 4 scores files:{licit,"
"spoof}/{dev,eval}")
......@@ -708,7 +715,7 @@ class FmrIapmr(PadPlot):
# for fmr.
fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
label = self._legends[idx] if self._legends is not None else \
'(%s/%s)' % (input_names[1], input_names[3])
'(%s/%s)' % (input_names[1], input_names[3])
if self._semilogx:
mpl.semilogx(fmr_list, iapmr_list, label=label)
else:
......@@ -717,7 +724,7 @@ class FmrIapmr(PadPlot):
def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures and
close pdf is needed '''
#only for plots
# only for plots
title = self._title if self._title is not None else "FMR vs IAPMR"
if title.replace(' ', ''):
mpl.title(title)
......@@ -733,7 +740,7 @@ class FmrIapmr(PadPlot):
self._pdf_page.savefig(fig)
#do not want to close PDF when running evaluate
# do not want to close PDF when running evaluate
if 'PdfPages' in self._ctx.meta and \
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
self._pdf_page.close()
bob/pad/base/script/fmr_iapmr.py
View file @ b63237ce
......@@ -6,6 +6,7 @@ from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.base.score import load
from . import figure
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='fmr_iapmr.pdf')
......
bob/pad/base/script/histograms.py
View file @ b63237ce
......@@ -6,6 +6,7 @@ from bob.extension.scripts.click_helper import (verbosity_option, bool_option)
from bob.bio.base.score import load
from . import figure
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.title_option()
......@@ -27,13 +28,13 @@ def hist(ctx, scores, evaluation, **kwargs):
""" Plots histograms of Bona fida and PA along with threshold
criterion.
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev scores
are provided, you must use flag `--no-evaluation`.
You need to provide one or more development score file(s) for each
experiment. You can also provide eval files along with dev files. If only
dev scores are provided, you must use flag `--no-evaluation`.
By default, when eval-scores are given, only eval-scores histograms are
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
computed from dev-scores. If you want to display dev-scores distributions
computed from dev-scores. If you want to display dev-scores distributions
as well, use ``--show-dev`` option.
Examples:
......@@ -47,6 +48,7 @@ def hist(ctx, scores, evaluation, **kwargs):
process = figure.HistPad(ctx, scores, evaluation, load.split)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1, min_arg=2)
@common_options.title_option()
......@@ -87,10 +89,10 @@ def vuln_hist(ctx, scores, evaluation, **kwargs):
See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
vulnerability analysis.
You need to provide one or more development score file(s) for each experiment.
You can also provide eval files along with dev files. If only dev-scores
are used set the flag `--no-evaluation`
is required in that case.
You need to provide one or more development score file(s) for each
experiment. You can also provide eval files along with dev files. If only
dev-scores are used set the flag `--no-evaluation` is required in that
case.
By default, when eval-scores are given, only eval-scores histograms are
displayed with threshold line
......
bob/pad/base/script/metrics.py
View file @ b63237ce
......@@ -7,6 +7,7 @@ from bob.extension.scripts.click_helper import (verbosity_option,
from bob.bio.base.score import load
from . import figure
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
......@@ -20,7 +21,8 @@ def metrics(ctx, scores, evaluation, **kwargs):
"""PAD ISO compliant metrics.
Reports several metrics based on a selected thresholds on the development
set and apply them on evaluation sets (if provided). The used thresholds are:
set and apply them on evaluation sets (if provided). The used thresholds
are:
bpcer20 When APCER is set to 5%.
......@@ -52,6 +54,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.eval_option()
......
bob/pad/base/script/pad.py
View file @ b63237ce
......@@ -8,5 +8,5 @@ from click_plugins import with_plugins
@with_plugins(pkg_resources.iter_entry_points('bob.pad.cli'))
@click.group()
def pad():
"""Entry for bob.pad commands."""
"""Presentation Attack Detection related commands."""
pass
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment