Commit bc9a2266 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

Add vuln_metrics and change command names

parent f276f395
......@@ -93,6 +93,34 @@ class Metrics(measure_figure.Metrics):
)
class MetricsVuln(measure_figure.Metrics):
def __init__(self, ctx, scores, evaluation, func_load):
super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)
''' Compute metrics from score files'''
def compute(self, idx, input_scores, input_names):
''' Compute metrics for the given criteria'''
neg_list, pos_list, _ = get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
criter = self._criterion or 'eer'
threshold = calc_threshold(criter, dev_neg, dev_pos) \
if self._thres is None else self._thres[idx]
far, frr = farfrr(neg_list[1], pos_list[1], threshold)
iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
title = self._legends[idx] if self._legends is not None else None
headers = ['' or title, '%s (threshold=%.2g)' %
(criter.upper(), threshold)]
rows = []
rows.append(['FMR (%)', '{:>5.1f}%'.format(100*far)])
rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr*100)])
rows.append(['HTER (%)', '{:>5.1f}%'.format(50*(far+frr))])
rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100*iapmr)])
click.echo(
tabulate(rows, headers, self._tablefmt),
file=self.log_file
)
class HistPad(measure_figure.Hist):
''' Histograms for PAD '''
......
......@@ -72,7 +72,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.style_option()
@verbosity_option()
@click.pass_context
def vuln(ctx, scores, evaluation, **kwargs):
def vuln_hist(ctx, scores, evaluation, **kwargs):
'''Vulnerability analysis distributions.
Plots the histogram of score distributions. You need to provide 4 score
......@@ -99,10 +99,10 @@ def vuln(ctx, scores, evaluation, **kwargs):
Examples:
$ bob pad vuln licit/scores-dev licit/scores-eval \
$ bob pad vuln_hist licit/scores-dev licit/scores-eval \
spoof/scores-dev spoof/scores-eval
$ bob pad vuln {licit,spoof}/scores-{dev,eval}
$ bob pad vuln_hist {licit,spoof}/scores-{dev,eval}
'''
process = figure.HistVuln(ctx, scores, evaluation, load.split)
process.run()
......@@ -51,3 +51,33 @@ def metrics(ctx, scores, evaluation, **kwargs):
"""
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.eval_option()
@common_options.table_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@open_file_mode_option()
@common_options.output_log_metric_option()
@common_options.legends_option()
@verbosity_option()
@click.pass_context
def vuln_metrics(ctx, scores, **kwargs):
"""Generate table of metrics for vulnerability PAD
You need to provide 2 or 4 scores
files for each PAD system in this order:
\b
* licit development scores
* licit evaluation scores
* spoof development scores
* spoof evaluation scores
Examples:
$ bob pad vuln_metrics {licit,spoof}/scores-{dev,eval}
"""
process = figure.MetricsVuln(ctx, scores, True, load.split)
process.run()
......@@ -140,8 +140,9 @@ setup(
# bob pad scripts
'bob.pad.cli': [
'metrics = bob.pad.base.script.metrics:metrics',
'vuln_metrics = bob.pad.base.script.metrics:vuln_metrics',
'hist = bob.pad.base.script.histograms:hist',
'vuln = bob.pad.base.script.histograms:vuln',
'vuln_hist = bob.pad.base.script.histograms:vuln_hist',
'det = bob.pad.base.script.det:det',
'epc = bob.pad.base.script.epc:epc',
'epsc = bob.pad.base.script.epc:epsc',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment