Skip to content
Snippets Groups Projects
Commit bc9a2266 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Add vuln_metrics and change command names

parent f276f395
No related branches found
No related tags found
2 merge requests!42Set of click commands for pad,!41Set of click commands for pad
...@@ -93,6 +93,34 @@ class Metrics(measure_figure.Metrics): ...@@ -93,6 +93,34 @@ class Metrics(measure_figure.Metrics):
) )
class MetricsVuln(measure_figure.Metrics):
def __init__(self, ctx, scores, evaluation, func_load):
super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
neg_list, pos_list, _ = get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
criter = self._criterion or 'eer'
threshold = calc_threshold(criter, dev_neg, dev_pos) \
if self._thres is None else self._thres[idx]
far, frr = farfrr(neg_list[1], pos_list[1], threshold)
iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
title = self._legends[idx] if self._legends is not None else None
headers = ['' or title, '%s (threshold=%.2g)' %
(criter.upper(), threshold)]
rows = []
rows.append(['FMR (%)', '{:>5.1f}%'.format(100*far)])
rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr*100)])
rows.append(['HTER (%)', '{:>5.1f}%'.format(50*(far+frr))])
rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100*iapmr)])
click.echo(
tabulate(rows, headers, self._tablefmt),
file=self.log_file
)
class HistPad(measure_figure.Hist): class HistPad(measure_figure.Hist):
''' Histograms for PAD ''' ''' Histograms for PAD '''
......
...@@ -72,7 +72,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -72,7 +72,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.style_option() @common_options.style_option()
@verbosity_option() @verbosity_option()
@click.pass_context @click.pass_context
def vuln(ctx, scores, evaluation, **kwargs): def vuln_hist(ctx, scores, evaluation, **kwargs):
'''Vulnerability analysis distributions. '''Vulnerability analysis distributions.
Plots the histogram of score distributions. You need to provide 4 score Plots the histogram of score distributions. You need to provide 4 score
...@@ -99,10 +99,10 @@ def vuln(ctx, scores, evaluation, **kwargs): ...@@ -99,10 +99,10 @@ def vuln(ctx, scores, evaluation, **kwargs):
Examples: Examples:
$ bob pad vuln licit/scores-dev licit/scores-eval \ $ bob pad vuln_hist licit/scores-dev licit/scores-eval \
spoof/scores-dev spoof/scores-eval spoof/scores-dev spoof/scores-eval
$ bob pad vuln {licit,spoof}/scores-{dev,eval} $ bob pad vuln_hist {licit,spoof}/scores-{dev,eval}
''' '''
process = figure.HistVuln(ctx, scores, evaluation, load.split) process = figure.HistVuln(ctx, scores, evaluation, load.split)
process.run() process.run()
...@@ -51,3 +51,33 @@ def metrics(ctx, scores, evaluation, **kwargs): ...@@ -51,3 +51,33 @@ def metrics(ctx, scores, evaluation, **kwargs):
""" """
process = figure.Metrics(ctx, scores, evaluation, load.split) process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run() process.run()
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.eval_option()
@common_options.table_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@open_file_mode_option()
@common_options.output_log_metric_option()
@common_options.legends_option()
@verbosity_option()
@click.pass_context
def vuln_metrics(ctx, scores, **kwargs):
"""Generate table of metrics for vulnerability PAD
You need to provide 2 or 4 scores
files for each PAD system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
Examples:
$ bob pad vuln_metrics {licit,spoof}/scores-{dev,eval}
"""
process = figure.MetricsVuln(ctx, scores, True, load.split)
process.run()
...@@ -140,8 +140,9 @@ setup( ...@@ -140,8 +140,9 @@ setup(
# bob pad scripts # bob pad scripts
'bob.pad.cli': [ 'bob.pad.cli': [
'metrics = bob.pad.base.script.metrics:metrics', 'metrics = bob.pad.base.script.metrics:metrics',
'vuln_metrics = bob.pad.base.script.metrics:vuln_metrics',
'hist = bob.pad.base.script.histograms:hist', 'hist = bob.pad.base.script.histograms:hist',
'vuln = bob.pad.base.script.histograms:vuln', 'vuln_hist = bob.pad.base.script.histograms:vuln_hist',
'det = bob.pad.base.script.det:det', 'det = bob.pad.base.script.det:det',
'epc = bob.pad.base.script.epc:epc', 'epc = bob.pad.base.script.epc:epc',
'epsc = bob.pad.base.script.epc:epsc', 'epsc = bob.pad.base.script.epc:epsc',
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment