From 7802eab33f46e1217f1d353a28ca9c2008b6a771 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 21 Jul 2020 08:27:37 +0200
Subject: [PATCH] [test.test_cli] Improve unit tests for significance script
---
bob/ip/binseg/engine/significance.py | 44 ++++++++++++++++------------
bob/ip/binseg/test/test_cli.py | 7 +++--
2 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/bob/ip/binseg/engine/significance.py b/bob/ip/binseg/engine/significance.py
index 54b2bb86..9fc64afd 100644
--- a/bob/ip/binseg/engine/significance.py
+++ b/bob/ip/binseg/engine/significance.py
@@ -6,6 +6,9 @@ import itertools
import textwrap
import multiprocessing
+import logging
+logger = logging.getLogger(__name__)
+
import h5py
from tqdm import tqdm
import numpy
@@ -806,31 +809,34 @@ def write_analysis_text(names, da, db, f):
# * The dependent variable should be approximately normally distributed. [!!!]
# * The dependent variable should not contain any outliers. [OK]
- f.write("\nPaired Significance Tests:\n")
+ if (diff == 0.0).all():
+ logger.error("Differences are exactly zero between both "
+ "patch distributions, for **all** samples. Statistical "
+ "significance tests are not meaningful in this context and "
+ "will be skipped. This typically indicates an issue with "
+ "the setup of prediction folders (duplicated?)")
+ return
+
+ f.write("\nPaired significance tests:\n")
w, p = scipy.stats.ttest_rel(da, db)
f.write(f" * Paired T (H0: same distro): S = {w:g}, p = {p:.5f}\n")
- try:
- f.write(" * Wilcoxon:\n")
+ f.write(" * Wilcoxon:\n")
- w, p = scipy.stats.wilcoxon(diff)
- f.write(f" * H0 = same distro: W = {w:g}, p = {p:.5f}\n")
+ w, p = scipy.stats.wilcoxon(diff)
+ f.write(f" * H0 = same distro: W = {w:g}, p = {p:.5f}\n")
- w, p = scipy.stats.wilcoxon(diff, alternative="greater")
- f.write(
- f" * H0 = med({names[0]}) < med({names[1]}): "
- f"W = {w:g}, p = {p:.5f}\n"
- )
+ w, p = scipy.stats.wilcoxon(diff, alternative="greater")
+ f.write(
+ f" * H0 = med({names[0]}) < med({names[1]}): "
+ f"W = {w:g}, p = {p:.5f}\n"
+ )
- w, p = scipy.stats.wilcoxon(diff, alternative="less")
- f.write(
- f" * H0 = med({names[0]}) > med({names[1]}): "
- f"W = {w:g}, p = {p:.5f}\n"
- )
- except ValueError as e:
- f.write(f" ERROR: Differences are exactly zero between both "
- f"patch distributions. The Wilcoxon test does not work in "
- f"these conditions (review your prediction directories): {e}\n")
+ w, p = scipy.stats.wilcoxon(diff, alternative="less")
+ f.write(
+ f" * H0 = med({names[0]}) > med({names[1]}): "
+ f"W = {w:g}, p = {p:.5f}\n"
+ )
def write_analysis_figures(names, da, db, fname):
diff --git a/bob/ip/binseg/test/test_cli.py b/bob/ip/binseg/test/test_cli.py
index b427185c..b27ec4a3 100644
--- a/bob/ip/binseg/test/test_cli.py
+++ b/bob/ip/binseg/test/test_cli.py
@@ -568,9 +568,10 @@ def _check_significance(runner):
keywords = {
r"^Evaluating patch 'accuracy' on": 2,
r"^Evaluating patch 'accuracy' differences on": 1,
- r"^#Samples/Median/Avg/Std.Dev.": 1,
- r"^Paired T-test": 1,
- r"^Wilcoxon test": 3,
+ #r"^Basic statistics from distributions:$": 1,
+ r"^Writing analysis figures": 1,
+ r"^Writing analysis summary": 1,
+ r"^Differences are exactly zero": 2,
}
buf.seek(0)
logging_output = buf.read()
--
GitLab