Commit 1020aa2e authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'metrics-fix' into 'master'

Fix the wrong total number reported in metrics

Closes bob.measure#61

See merge request !66
parents 67a3c188 05331f85
Pipeline #36723 passed with stages
in 10 minutes and 34 seconds
...@@ -50,7 +50,7 @@ def metrics_option( ...@@ -50,7 +50,7 @@ def metrics_option(
name="metrics", name="metrics",
help="List of metrics to print. Provide a string with comma separated metric " help="List of metrics to print. Provide a string with comma separated metric "
"names. For possible values see the default value.", "names. For possible values see the default value.",
default="apcer_pais,apcer,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score", default="apcer_pais,apcer_ap,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score",
**kwargs **kwargs
): ):
"""The metrics option""" """The metrics option"""
...@@ -143,13 +143,13 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs): ...@@ -143,13 +143,13 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
@common_options.metrics_command( @common_options.metrics_command(
common_options.METRICS_HELP.format( common_options.METRICS_HELP.format(
names="FtA, APCER, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score", names="FtA, APCER_AP, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
criteria=CRITERIA, criteria=CRITERIA,
score_format=SCORE_FORMAT, score_format=SCORE_FORMAT,
hter_note="Note that APCER = max(APCER_pais), BPCER=FNR, " hter_note="Note that APCER_AP = max(APCER_pais), BPCER=FNR, "
"FAR = FPR * (1 - FtA), " "FAR = FPR * (1 - FtA), "
"FRR = FtA + FNR * (1 - FtA), " "FRR = FtA + FNR * (1 - FtA), "
"ACER = (APCER + BPCER) / 2, " "ACER = (APCER_AP + BPCER) / 2, "
"and HTER = (FPR + FNR) / 2. " "and HTER = (FPR + FNR) / 2. "
"You can control which metrics are printed using the --metrics option. " "You can control which metrics are printed using the --metrics option. "
"You can use --regexps and --regexp_column options to change the behavior " "You can use --regexps and --regexp_column options to change the behavior "
...@@ -160,7 +160,7 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs): ...@@ -160,7 +160,7 @@ def gen(ctx, outdir, mean_match, mean_non_match, n_sys, **kwargs):
epilog="""\b epilog="""\b
More Examples: More Examples:
\b \b
bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer,bpcer,acer,hter \ bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer_ap,bpcer,acer,hter \
/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval} /scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}
See also ``bob pad multi-metrics``. See also ``bob pad multi-metrics``.
...@@ -241,7 +241,7 @@ See also ``bob pad metrics``. ...@@ -241,7 +241,7 @@ See also ``bob pad metrics``.
) )
@regexps_option() @regexps_option()
@regexp_column_option() @regexp_column_option()
@metrics_option(default="fta,apcer_pais,apcer,bpcer,acer,hter") @metrics_option(default="fta,apcer_pais,apcer_ap,bpcer,acer,hter")
def multi_metrics( def multi_metrics(
ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs ctx, scores, evaluation, protocols_number, regexps, regexp_column, metrics, **kwargs
): ):
......
...@@ -47,9 +47,9 @@ class Metrics(bio_figure.Metrics): ...@@ -47,9 +47,9 @@ class Metrics(bio_figure.Metrics):
def _numbers(self, threshold, pos, negs, all_negs, fta): def _numbers(self, threshold, pos, negs, all_negs, fta):
pais = list(negs.keys()) pais = list(negs.keys())
apcer_pais, apcer, bpcer = apcer_bpcer(threshold, pos, *[negs[k] for k in pais]) apcer_pais, apcer_ap, bpcer = apcer_bpcer(threshold, pos, *[negs[k] for k in pais])
apcer_pais = {k: apcer_pais[i] for i, k in enumerate(pais)} apcer_pais = {k: apcer_pais[i] for i, k in enumerate(pais)}
acer = (apcer + bpcer) / 2.0 acer = (apcer_ap + bpcer) / 2.0
fpr, fnr = farfrr(all_negs, pos, threshold) fpr, fnr = farfrr(all_negs, pos, threshold)
hter = (fpr + fnr) / 2.0 hter = (fpr + fnr) / 2.0
far = fpr * (1 - fta) far = fpr * (1 - fta)
...@@ -67,7 +67,7 @@ class Metrics(bio_figure.Metrics): ...@@ -67,7 +67,7 @@ class Metrics(bio_figure.Metrics):
f1_score = f_score(all_negs, pos, threshold, 1) f1_score = f_score(all_negs, pos, threshold, 1)
metrics = dict( metrics = dict(
apcer_pais=apcer_pais, apcer_pais=apcer_pais,
apcer=apcer, apcer_ap=apcer_ap,
bpcer=bpcer, bpcer=bpcer,
acer=acer, acer=acer,
fta=fta, fta=fta,
...@@ -98,7 +98,7 @@ class Metrics(bio_figure.Metrics): ...@@ -98,7 +98,7 @@ class Metrics(bio_figure.Metrics):
metrics[k] = "%s%% (%d/%d)" % ( metrics[k] = "%s%% (%d/%d)" % (
format(100 * v, n_dec), format(100 * v, n_dec),
metrics["fp" if k == "fpr" else "fn"], metrics["fp" if k == "fpr" else "fn"],
metrics["np" if k == "fpr" else "nn"], metrics["nn" if k == "fpr" else "np"],
) )
else: else:
metrics[k] = "%s%%" % format(100 * v, n_dec) metrics[k] = "%s%%" % format(100 * v, n_dec)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment