[engine.evaluator] Dump scores for patches as well

bob/ip/binseg/engine/evaluator.py

@@ -21,6 +21,9 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+_PATCH_CONFIG = (128, 128, 32)
+"""Stock configuration for patch analysis"""
+
 
 def _posneg(pred, gt, threshold):
     """Calculates true and false positives and negatives"""
@@ -51,7 +54,7 @@ def _posneg(pred, gt, threshold):
 
 def _sample_measures(pred, gt, steps):
     """
-    Calculates measures on one single sample and saves it to disk
+    Calculates measures on one single sample
 
 
     Parameters
@@ -65,7 +68,7 @@ def _sample_measures(pred, gt, steps):
 
     steps : int
         number of steps to use for threshold analysis.  The step size is
-        calculated from this by dividing ``1.0/steps``.
+        calculated from this by dividing ``1.0/steps``
 
 
     Returns
@@ -136,6 +139,62 @@ def _sample_measures(pred, gt, steps):
     )
 
 
+def _patch_measures(pred, gt, steps, size):
+    """
+    Calculates measures on patches of a single sample
+
+
+    Parameters
+    ----------
+
+    pred : torch.Tensor
+        pixel-wise predictions
+
+    gt : torch.Tensor
+        ground-truth (annotations)
+
+    steps : int
+        number of steps to use for threshold analysis.  The step size is
+        calculated from this by dividing ``1.0/steps``
+
+    size : :py:class:`tuple`
+        A tripplet with three integers indicating the height, width, and stride
+        of patches to break measure analysis into.  In this case, the input
+        image and ground-truth will be cut into blocks of the provided height
+        and width, overlapping by the total overlap size, starting on the top
+        left corner and then moving right and to the bottom.  Windows on the
+        left and bottom edge of the image may be incomplete.
+
+
+    Returns
+    -------
+
+    measures : pandas.DataFrame
+
+        A pandas dataframe with the following columns:
+
+        * patch: int
+        * threshold: float
+        * precision: float
+        * recall: float
+        * specificity: float
+        * accuracy: float
+        * jaccard: float
+        * f1_score: float
+
+    """
+
+    height, width, stride = window_size
+    pred_patches = pred.unfold(0, height, stride).unfold(1, width, stride)
+    gt_patches = unfold(0, height, stride).unfold(1, width, stride)
+
+    # add patch number for each set of measures
+    dfs = [_sample_measures(p, g, step) for p,g in zip(pred_patches, gt_patches)]
+    for i, k in enumerate(dfs): k['patch'] = i
+
+    return pandas.concat(dfs, ignore_index=True)
+
+
 def _sample_analysis(
     img,
     pred,
@@ -294,6 +353,12 @@ def run(
             os.makedirs(os.path.dirname(fullpath), exist_ok=True)
             data[stem].to_csv(fullpath)
 
+            # saves patch analysis
+            fullpath = os.path.join(output_folder, name, "patches", f"{stem}.csv")
+            tqdm.write(f"Saving {fullpath}...")
+            os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+            _patch_measures(pred, gt, steps, _PATCH_CONFIG).to_csv(fullpath)
+
         if overlayed_folder is not None:
             overlay_image = _sample_analysis(
                 image, pred, gt, threshold=threshold, overlay=True
@@ -422,6 +487,13 @@ def compare_annotators(baseline, other, name, output_folder,
             os.makedirs(os.path.dirname(fullpath), exist_ok=True)
             data[stem].to_csv(fullpath)
 
+            # saves patch analysis
+            fullpath = os.path.join(output_folder, "second-annotator", name,
+                    "patches", f"{stem}.csv")
+            tqdm.write(f"Saving {fullpath}...")
+            os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+            _patch_measures(pred, gt, 2, _PATCH_CONFIG).to_csv(fullpath)
+
         if overlayed_folder is not None:
             overlay_image = _sample_analysis(
                 image, pred, gt, threshold=0.5, overlay=True

bob/ip/binseg/test/test_cli.py

@@ -151,6 +151,11 @@ def _check_experiment_stare(overlay):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(traindir), "*.csv")), 10
         )
+        traindir = os.path.join(eval_folder, "train", "patches", "stare-images")
+        assert os.path.exists(traindir)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(traindir), "*.csv")), 10
+        )
 
         assert os.path.exists(os.path.join(eval_folder, "test.csv"))
         # checks individual performance figures are there
@@ -159,6 +164,11 @@ def _check_experiment_stare(overlay):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
         )
+        testdir = os.path.join(eval_folder, "test", "patches", "stare-images")
+        assert os.path.exists(testdir)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
+        )
 
         assert os.path.exists(
             os.path.join(eval_folder, "second-annotator", "train.csv")
@@ -170,6 +180,12 @@ def _check_experiment_stare(overlay):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(traindir_sa), "*.csv")), 10
         )
+        traindir_sa = os.path.join(eval_folder, "second-annotator", "patches",
+                "train", "stare-images")
+        assert os.path.exists(traindir_sa)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(traindir_sa), "*.csv")), 10
+        )
 
         assert os.path.exists(
             os.path.join(eval_folder, "second-annotator", "test.csv")
@@ -180,6 +196,12 @@ def _check_experiment_stare(overlay):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
         )
+        testdir_sa = os.path.join(eval_folder, "second-annotator", "patches",
+                "test", "stare-images")
+        assert os.path.exists(testdir_sa)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
+        )
 
         overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
         traindir = os.path.join(overlay_folder, "train", "stare-images")
@@ -439,6 +461,11 @@ def _check_evaluate(runner):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
         )
+        testdir = os.path.join(output_folder, "test", "patches", "stare-images")
+        assert os.path.exists(testdir)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(testdir), "*.csv")), 10
+        )
 
         assert os.path.exists(
             os.path.join(output_folder, "second-annotator", "test.csv")
@@ -450,6 +477,12 @@ def _check_evaluate(runner):
         nose.tools.eq_(
             len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
         )
+        testdir_sa = os.path.join(output_folder, "second-annotator", "test",
+                "patches", "stare-images")
+        assert os.path.exists(testdir_sa)
+        nose.tools.eq_(
+            len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")), 10
+        )
 
         # check overlayed images are there (since we requested them)
         basedir = os.path.join(overlay_folder, "test", "stare-images")