Commit d27974c7 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

move code around

parent 9e51851e
Pipeline #21208 failed with stage
in 32 minutes and 2 seconds
...@@ -5,6 +5,44 @@ ...@@ -5,6 +5,44 @@
import bob.measure import bob.measure
import numpy import numpy
from bob.measure import (
far_threshold, eer_threshold, min_hter_threshold)
def calc_threshold(method, neg, pos):
"""Calculates the threshold based on the given method.
The scores should be sorted!
Parameters
----------
method : str
One of ``bpcer20``, ``eer``, ``min-hter``.
neg : array_like
The negative scores. They should be sorted!
pos : array_like
The positive scores. They should be sorted!
Returns
-------
float
The calculated threshold.
Raises
------
ValueError
If method is unknown.
"""
method = method.lower()
if method == 'bpcer20':
threshold = far_threshold(neg, pos, 0.05, True)
elif method == 'eer':
threshold = eer_threshold(neg, pos, True)
elif method == 'min-hter':
threshold = min_hter_threshold(neg, pos, True)
else:
raise ValueError("Unknown threshold criteria: {}".format(method))
return threshold
def calc_pass_rate(threshold, attacks): def calc_pass_rate(threshold, attacks):
......
This diff is collapsed.
'''Runs error analysis on score sets, outputs metrics and plots'''
import bob.measure.script.figure as measure_figure
import bob.bio.base.script.figure as bio_figure
from .error_utils import calc_threshold
ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')
class Metrics(measure_figure.Metrics):
'''Compute metrics from score files'''
def __init__(self, ctx, scores, evaluation, func_load):
super(Metrics, self).__init__(
ctx, scores, evaluation, func_load,
names=('FtA', 'APCER', 'BPCER', 'FAR', 'FRR', 'ACER'))
def get_thres(self, criterion, dev_neg, dev_pos, far):
if self._criterion == 'bpcer20':
return calc_threshold('bpcer20', dev_neg, dev_pos)
else:
return super(Metrics, self).get_thres(
criterion, dev_neg, dev_pos, far)
class Roc(bio_figure.Roc):
'''ROC for PAD'''
def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = ctx.meta.get('x_label') or 'APCER'
self._y_label = ctx.meta.get('y_label') or '1 - BPCER'
class Det(bio_figure.Det):
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = ctx.meta.get('x_label') or 'APCER (%)'
self._y_label = ctx.meta.get('y_label') or 'BPCER (%)'
class Hist(measure_figure.Hist):
''' Histograms for PAD '''
def _setup_hist(self, neg, pos):
self._title_base = 'PAD'
self._density_hist(
pos[0], n=0, label='Bona Fide', color='C1'
)
self._density_hist(
neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
hatch='\\\\'
)
...@@ -13,89 +13,11 @@ from bob.measure import ( ...@@ -13,89 +13,11 @@ from bob.measure import (
from bob.measure.plot import (det, det_axis, roc_for_far, log_values) from bob.measure.plot import (det, det_axis, roc_for_far, log_values)
from . import error_utils from . import error_utils
ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')
def calc_threshold(method, neg, pos):
"""Calculates the threshold based on the given method.
The scores should be sorted!
Parameters
----------
method : str
One of ``bpcer20``, ``eer``, ``min-hter``.
neg : array_like
The negative scores. They should be sorted!
pos : array_like
The positive scores. They should be sorted!
Returns
-------
float
The calculated threshold.
Raises
------
ValueError
If method is unknown.
"""
method = method.lower()
if method == 'bpcer20':
threshold = far_threshold(neg, pos, 0.05, True)
elif method == 'eer':
threshold = eer_threshold(neg, pos, True)
elif method == 'min-hter':
threshold = min_hter_threshold(neg, pos, True)
else:
raise ValueError("Unknown threshold criteria: {}".format(method))
return threshold
class Metrics(measure_figure.Metrics): class Metrics(measure_figure.Metrics):
'''Compute metrics from score files'''
def __init__(self, ctx, scores, evaluation, func_load): def __init__(self, ctx, scores, evaluation, func_load):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load) super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
neg_list, pos_list, _ = get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
dev_file = input_names[0]
if self._eval:
eval_neg, eval_pos = neg_list[1], pos_list[1]
eval_file = input_names[1]
title = self._legends[idx] if self._legends is not None else None
headers = ['' or title, 'Development %s' % dev_file]
if self._eval:
headers.append('Eval. % s' % eval_file)
for m in ALL_CRITERIA:
raws = []
threshold = calc_threshold(m, dev_neg, dev_pos)
click.echo("\nThreshold of %f selected with the %s criteria" % (
threshold, m), file=self.log_file)
apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)
raws.append(['APCER', '{:>5.1f}%'.format(apcer * 100)])
raws.append(['BPCER', '{:>5.1f}%'.format(bpcer * 100)])
raws.append(['ACER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])
if self._eval and eval_neg is not None:
apcer, bpcer = farfrr(eval_neg, eval_pos, threshold)
raws[0].append('{:>5.1f}%'.format(apcer * 100))
raws[1].append('{:>5.1f}%'.format(bpcer * 100))
raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50))
click.echo(
tabulate(raws, headers, self._tablefmt),
file=self.log_file
)
class MetricsVuln(measure_figure.Metrics):
def __init__(self, ctx, scores, evaluation, func_load):
super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files''' ''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names): def compute(self, idx, input_scores, input_names):
...@@ -121,20 +43,6 @@ class MetricsVuln(measure_figure.Metrics): ...@@ -121,20 +43,6 @@ class MetricsVuln(measure_figure.Metrics):
) )
class HistPad(measure_figure.Hist):
''' Histograms for PAD '''
def _setup_hist(self, neg, pos):
self._title_base = 'PAD'
self._density_hist(
pos[0], n=0, label='Bona Fide', color='C1'
)
self._density_hist(
neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
hatch='\\\\'
)
def _iapmr_dot(threshold, iapmr, real_data, **kwargs): def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
# plot a dot on threshold versus IAPMR line and show IAPMR as a number # plot a dot on threshold versus IAPMR line and show IAPMR as a number
axlim = mpl.axis() axlim = mpl.axis()
...@@ -176,6 +84,10 @@ def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs): ...@@ -176,6 +84,10 @@ def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
class HistVuln(measure_figure.Hist): class HistVuln(measure_figure.Hist):
''' Histograms for vulnerability ''' ''' Histograms for vulnerability '''
def __init__(self, ctx, scores, evaluation, func_load):
super(HistVuln, self).__init__(
ctx, scores, evaluation, func_load, nhist_per_system=3)
def _setup_hist(self, neg, pos): def _setup_hist(self, neg, pos):
self._title_base = 'Vulnerability' self._title_base = 'Vulnerability'
self._density_hist( self._density_hist(
...@@ -519,28 +431,13 @@ class Epsc3D(Epsc): ...@@ -519,28 +431,13 @@ class Epsc3D(Epsc):
self._pdf_page.savefig() self._pdf_page.savefig()
class Roc(bio_figure.Roc): class BaseVulnDetRoc(PadPlot):
'''ROC for PAD'''
def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = ctx.meta.get('x_label') or 'APCER'
self._y_label = ctx.meta.get('y_label') or '1 - BPCER'
class DetPad(bio_figure.Det):
def __init__(self, ctx, scores, evaluation, func_load):
super(DetPad, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = ctx.meta.get('x_label') or 'APCER'
self._y_label = ctx.meta.get('y_label') or 'BPCER'
class BaseDetRoc(PadPlot):
'''Base for DET and ROC''' '''Base for DET and ROC'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
no_spoof): no_spoof):
super(BaseDetRoc, self).__init__(ctx, scores, evaluation, func_load) super(BaseVulnDetRoc, self).__init__(
ctx, scores, evaluation, func_load)
self._no_spoof = no_spoof self._no_spoof = no_spoof
self._criteria = criteria or 'eer' self._criteria = criteria or 'eer'
self._real_data = True if real_data is None else real_data self._real_data = True if real_data is None else real_data
...@@ -719,7 +616,7 @@ class BaseDetRoc(PadPlot): ...@@ -719,7 +616,7 @@ class BaseDetRoc(PadPlot):
pass pass
class Det(BaseDetRoc): class DetVuln(BaseVulnDetRoc):
'''DET for vuln''' '''DET for vuln'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
...@@ -753,7 +650,7 @@ class Det(BaseDetRoc): ...@@ -753,7 +650,7 @@ class Det(BaseDetRoc):
) )
class RocVuln(BaseDetRoc): class RocVuln(BaseVulnDetRoc):
'''ROC for vuln''' '''ROC for vuln'''
def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment