[engine,script] Simplify implementation of subset folder structure; Fix unit tests

bob/ip/binseg/engine/evaluator.py

@@ -270,11 +270,15 @@ def run(
     # Collect overall measures
     data = {}
 
+    use_predictions_folder = os.path.join(predictions_folder, name)
+    if not os.path.exists(use_predictions_folder):
+        use_predictions_folder = predictions_folder
+
     for sample in tqdm(dataset):
         stem = sample[0]
         image = sample[1]
         gt = sample[2]
-        pred_fullpath = os.path.join(predictions_folder, stem + ".hdf5")
+        pred_fullpath = os.path.join(use_predictions_folder, stem + ".hdf5")
         with h5py.File(pred_fullpath, "r") as f:
             pred = f["array"][:]
         pred = torch.from_numpy(pred)
@@ -288,7 +292,7 @@ def run(
             overlay_image = _sample_analysis(
                 image, pred, gt, threshold=threshold, overlay=True
             )
-            fullpath = os.path.join(overlayed_folder, f"{stem}.png")
+            fullpath = os.path.join(overlayed_folder, name, f"{stem}.png")
             tqdm.write(f"Saving {fullpath}...")
             os.makedirs(os.path.dirname(fullpath), exist_ok=True)
             overlay_image.save(fullpath)
@@ -409,7 +413,7 @@ def compare_annotators(baseline, other, name, output_folder,
                 image, pred, gt, threshold=0.5, overlay=True
             )
             fullpath = os.path.join(overlayed_folder, "second-annotator",
-                    f"{stem}.png")
+                    name, f"{stem}.png")
             tqdm.write(f"Saving {fullpath}...")
             os.makedirs(os.path.dirname(fullpath), exist_ok=True)
             overlay_image.save(fullpath)

bob/ip/binseg/engine/predictor.py

@@ -101,7 +101,7 @@ def _save_overlayed_png(stem, image, prob, output_folder):
     _save_image(stem, ".png", overlayed_image(image, prob), output_folder)
 
 
-def run(model, data_loader, device, output_folder, overlayed_folder):
+def run(model, data_loader, name, device, output_folder, overlayed_folder):
     """
     Runs inference on input data, outputs HDF5 files with predictions
 
@@ -112,6 +112,10 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
 
     data_loader : py:class:`torch.torch.utils.data.DataLoader`
 
+    name : str
+        the local name of this dataset (e.g. ``train``, or ``test``), to be
+        used when saving measures files.
+
     device : str
         device to use ``cpu`` or ``cuda:0``
 
@@ -138,9 +142,14 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
     times = []
     len_samples = []
 
-    for samples in tqdm(
-        data_loader, desc="batches", leave=False, disable=None,
-    ):
+    output_folder = os.path.join(output_folder, name)
+    overlayed_folder = (
+        os.path.join(overlayed_folder, name)
+        if overlayed_folder is not None
+        else overlayed_folder
+    )
+
+    for samples in tqdm(data_loader, desc="batches", leave=False, disable=None):
 
         names = samples[0]
         images = samples[1].to(

bob/ip/binseg/script/evaluate.py

@@ -45,38 +45,6 @@ def _validate_threshold(t, dataset):
     return t
 
 
-def _get_folder(folder, name):
-    """Guesses the prediction folder to use based on the dataset name
-
-    This function will look for ``folder/name`` if it exists, and
-    return this.  Otherwise defaults to ``folder``.
-
-
-    Parameters
-    ==========
-
-    folder : str
-        Path to the root of the predictions folder
-
-    name : str
-        The name of the dataset for which we are trying to find the predictions
-        folder
-
-
-    Returns
-    =======
-
-    path : str
-        The best path to use as the root of the predictions folder for this
-        dataset.
-
-    """
-    candidate = os.path.join(folder, name)
-    if os.path.exists(candidate):
-        return candidate
-    return folder
-
-
 @click.command(
     entry_point_group="bob.ip.binseg.config",
     cls=ConfigCommand,
@@ -208,13 +176,14 @@ def evaluate(
         # first run evaluation for reference dataset, do not save overlays
         logger.info(f"Evaluating threshold on '{threshold}' set")
         threshold = run(
-            dataset[threshold],
-            threshold,
-            _get_folder(predictions_folder, threshold),
-            steps=steps,
+            dataset[threshold], threshold, predictions_folder, steps=steps
         )
         logger.info(f"Set --threshold={threshold:.5f}")
 
+    # clean-up the overlayed path
+    if overlayed is not None:
+        overlayed = overlayed.strip()
+
     # now run with the
     for k, v in dataset.items():
         if k.startswith("_"):
@@ -224,7 +193,7 @@ def evaluate(
         run(
             v,
             k,
-            _get_folder(predictions_folder, k),
+            predictions_folder,
             output_folder,
             overlayed,
             threshold,

bob/ip/binseg/script/predict.py

@@ -145,8 +145,4 @@ def predict(output_folder, model, dataset, batch_size, device, weight,
             shuffle=False,
             pin_memory=torch.cuda.is_available(),
         )
-        # this avoids collisions if we have, e.g., multi-resolution versions
-        # of the same dataset being evaluated, or datasets for which filenames
-        # may match.
-        use_output_folder = os.path.join(output_folder, k)
-        run(model, data_loader, device, use_output_folder, overlayed)
+        run(model, data_loader, k, device, output_folder, overlayed)

bob/ip/binseg/test/test_cli.py

@@ -117,20 +117,30 @@ def _check_experiment_stare(overlay):
 
         # check predictions are there
         predict_folder = os.path.join(output_folder, "predictions")
-        basedir = os.path.join(predict_folder, "stare-images")
-        assert os.path.exists(basedir)
-        nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 20)
+        traindir = os.path.join(predict_folder, "train", "stare-images")
+        assert os.path.exists(traindir)
+        nose.tools.eq_(len(fnmatch.filter(os.listdir(traindir), "*.hdf5")), 10)
+        testdir = os.path.join(predict_folder, "test", "stare-images")
+        assert os.path.exists(testdir)
+        nose.tools.eq_(len(fnmatch.filter(os.listdir(testdir), "*.hdf5")), 10)
 
         overlay_folder = os.path.join(output_folder, "overlayed", "predictions")
-        basedir = os.path.join(overlay_folder, "stare-images")
+        traindir = os.path.join(overlay_folder, "train", "stare-images")
+        testdir = os.path.join(overlay_folder, "test", "stare-images")
         if overlay:
             # check overlayed images are there (since we requested them)
-            assert os.path.exists(basedir)
+            assert os.path.exists(traindir)
+            nose.tools.eq_(
+                len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+            )
+            # check overlayed images are there (since we requested them)
+            assert os.path.exists(testdir)
             nose.tools.eq_(
-                len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+                len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
             )
         else:
-            assert not os.path.exists(basedir)
+            assert not os.path.exists(traindir)
+            assert not os.path.exists(testdir)
 
         # check evaluation outputs
         eval_folder = os.path.join(output_folder, "analysis")
@@ -144,29 +154,41 @@ def _check_experiment_stare(overlay):
         )
 
         overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
-        basedir = os.path.join(overlay_folder, "stare-images")
+        traindir = os.path.join(overlay_folder, "train", "stare-images")
+        testdir = os.path.join(overlay_folder, "test", "stare-images")
         if overlay:
             # check overlayed images are there (since we requested them)
-            assert os.path.exists(basedir)
+            assert os.path.exists(traindir)
             nose.tools.eq_(
-                len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+                len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+            )
+            assert os.path.exists(testdir)
+            nose.tools.eq_(
+                len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
             )
         else:
-            assert not os.path.exists(basedir)
+            assert not os.path.exists(traindir)
+            assert not os.path.exists(testdir)
 
         # check overlayed images from first-to-second annotator comparisons
         # are there (since we requested them)
         overlay_folder = os.path.join(
             output_folder, "overlayed", "analysis", "second-annotator"
         )
-        basedir = os.path.join(overlay_folder, "stare-images")
+        traindir = os.path.join(overlay_folder, "train", "stare-images")
+        testdir = os.path.join(overlay_folder, "test", "stare-images")
         if overlay:
-            assert os.path.exists(basedir)
+            assert os.path.exists(traindir)
             nose.tools.eq_(
-                len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+                len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+            )
+            assert os.path.exists(testdir)
+            nose.tools.eq_(
+                len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
             )
         else:
-            assert not os.path.exists(basedir)
+            assert not os.path.exists(traindir)
+            assert not os.path.exists(testdir)
 
         # check outcomes of the comparison phase
         assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))
@@ -388,7 +410,7 @@ def _check_evaluate(runner):
         )
 
         # check overlayed images are there (since we requested them)
-        basedir = os.path.join(overlay_folder, "stare-images")
+        basedir = os.path.join(overlay_folder, "test", "stare-images")
         assert os.path.exists(basedir)
         nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)