From c9d5d42c14890c485fb2adf6fa94dc685e1f8d64 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Fri, 26 Jun 2020 15:28:51 +0200
Subject: [PATCH] [engine.evaluator] Dump scores for patches as well
---
bob/ip/binseg/engine/evaluator.py | 76 ++++++++++++++++++++++++++++++-
bob/ip/binseg/test/test_cli.py | 33 ++++++++++++++
2 files changed, 107 insertions(+), 2 deletions(-)
diff --git a/bob/ip/binseg/engine/evaluator.py b/bob/ip/binseg/engine/evaluator.py
index 8667300e..1b5be9f7 100644
--- a/bob/ip/binseg/engine/evaluator.py
+++ b/bob/ip/binseg/engine/evaluator.py
@@ -21,6 +21,9 @@ import logging
logger = logging.getLogger(__name__)
+_PATCH_CONFIG = (128, 128, 32)
+"""Stock configuration for patch analysis"""
+
def _posneg(pred, gt, threshold):
"""Calculates true and false positives and negatives"""
@@ -51,7 +54,7 @@ def _posneg(pred, gt, threshold):
def _sample_measures(pred, gt, steps):
"""
- Calculates measures on one single sample and saves it to disk
+ Calculates measures on one single sample
Parameters
@@ -65,7 +68,7 @@ def _sample_measures(pred, gt, steps):
steps : int
number of steps to use for threshold analysis. The step size is
- calculated from this by dividing ``1.0/steps``.
+ calculated from this by dividing ``1.0/steps``
Returns
@@ -136,6 +139,62 @@ def _sample_measures(pred, gt, steps):
)
+def _patch_measures(pred, gt, steps, size):
+ """
+ Calculates measures on patches of a single sample
+
+
+ Parameters
+ ----------
+
+ pred : torch.Tensor
+ pixel-wise predictions
+
+ gt : torch.Tensor
+ ground-truth (annotations)
+
+ steps : int
+ number of steps to use for threshold analysis. The step size is
+ calculated from this by dividing ``1.0/steps``
+
+ size : :py:class:`tuple`
+ A tripplet with three integers indicating the height, width, and stride
+ of patches to break measure analysis into. In this case, the input
+ image and ground-truth will be cut into blocks of the provided height
+ and width, overlapping by the total overlap size, starting on the top
+ left corner and then moving right and to the bottom. Windows on the
+ left and bottom edge of the image may be incomplete.
+
+
+ Returns
+ -------
+
+ measures : pandas.DataFrame
+
+ A pandas dataframe with the following columns:
+
+ * patch: int
+ * threshold: float
+ * precision: float
+ * recall: float
+ * specificity: float
+ * accuracy: float
+ * jaccard: float
+ * f1_score: float
+
+ """
+
+ height, width, stride = window_size
+ pred_patches = pred.unfold(0, height, stride).unfold(1, width, stride)
+ gt_patches = unfold(0, height, stride).unfold(1, width, stride)
+
+ # add patch number for each set of measures
+ dfs = [_sample_measures(p, g, step) for p,g in zip(pred_patches, gt_patches)]
+ for i, k in enumerate(dfs): k['patch'] = i
+
+ return pandas.concat(dfs, ignore_index=True)
+
+
def _sample_analysis(
img,
pred,
@@ -294,6 +353,12 @@ def run(
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
data[stem].to_csv(fullpath)
+ # saves patch analysis
+ fullpath = os.path.join(output_folder, name, "patches", f"{stem}.csv")
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ _patch_measures(pred, gt, steps, _PATCH_CONFIG).to_csv(fullpath)
+
if overlayed_folder is not None:
overlay_image = _sample_analysis(
image, pred, gt, threshold=threshold, overlay=True
@@ -422,6 +487,13 @@ def compare_annotators(baseline, other, name, output_folder,
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
data[stem].to_csv(fullpath)
+ # saves patch analysis
+ fullpath = os.path.join(output_folder, "second-annotator", name,
+ "patches", f"{stem}.csv")
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ _patch_measures(pred, gt, 2, _PATCH_CONFIG).to_csv(fullpath)
+
if overlayed_folder is not None:
overlay_image = _sample_analysis(
image, pred, gt, threshold=0.5, overlay=True
diff --git a/bob/ip/binseg/test/test_cli.py b/bob/ip/binseg/test/test_cli.py
index 97f3dc25..2949d86f 100644
--- a/bob/ip/binseg/test/test_cli.py
+++ b/bob/ip/binseg/test/test_cli.py
@@ -151,6 +151,11 @@ def _check_experiment_stare(overlay):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(traindir), "*.csv")), 10
)
+ traindir = os.path.join(eval_folder, "train", "patches", "stare-images")
+ assert os.path.exists(traindir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(traindir), "*.csv")), 10
+ )
assert os.path.exists(os.path.join(eval_folder, "test.csv"))
# checks individual performance figures are there
@@ -159,6 +164,11 @@ def _check_experiment_stare(overlay):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
)
+ testdir = os.path.join(eval_folder, "test", "patches", "stare-images")
+ assert os.path.exists(testdir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
+ )
assert os.path.exists(
os.path.join(eval_folder, "second-annotator", "train.csv")
@@ -170,6 +180,12 @@ def _check_experiment_stare(overlay):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(traindir_sa), "*.csv")), 10
)
+ traindir_sa = os.path.join(eval_folder, "second-annotator", "patches",
+ "train", "stare-images")
+ assert os.path.exists(traindir_sa)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(traindir_sa), "*.csv")), 10
+ )
assert os.path.exists(
os.path.join(eval_folder, "second-annotator", "test.csv")
@@ -180,6 +196,12 @@ def _check_experiment_stare(overlay):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
)
+ testdir_sa = os.path.join(eval_folder, "second-annotator", "patches",
+ "test", "stare-images")
+ assert os.path.exists(testdir_sa)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
+ )
overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
traindir = os.path.join(overlay_folder, "train", "stare-images")
@@ -439,6 +461,11 @@ def _check_evaluate(runner):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
)
+ testdir = os.path.join(output_folder, "test", "patches", "stare-images")
+ assert os.path.exists(testdir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
+ )
assert os.path.exists(
os.path.join(output_folder, "second-annotator", "test.csv")
@@ -450,6 +477,12 @@ def _check_evaluate(runner):
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
)
+ testdir_sa = os.path.join(output_folder, "second-annotator", "test",
+ "patches", "stare-images")
+ assert os.path.exists(testdir_sa)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
+ )
# check overlayed images are there (since we requested them)
basedir = os.path.join(overlay_folder, "test", "stare-images")
--
GitLab