From 1462b43ba9b213617e84d7ca95602270a8f6cf37 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Wed, 20 May 2020 18:02:10 +0200
Subject: [PATCH] [engine,script] Simplify implementation of subset folder
structure; Fix unit tests
---
bob/ip/binseg/engine/evaluator.py | 10 ++++--
bob/ip/binseg/engine/predictor.py | 17 +++++++---
bob/ip/binseg/script/evaluate.py | 43 ++++--------------------
bob/ip/binseg/script/predict.py | 6 +---
bob/ip/binseg/test/test_cli.py | 54 ++++++++++++++++++++++---------
5 files changed, 65 insertions(+), 65 deletions(-)
diff --git a/bob/ip/binseg/engine/evaluator.py b/bob/ip/binseg/engine/evaluator.py
index 66d5d40b..8d3b1195 100644
--- a/bob/ip/binseg/engine/evaluator.py
+++ b/bob/ip/binseg/engine/evaluator.py
@@ -270,11 +270,15 @@ def run(
# Collect overall measures
data = {}
+ use_predictions_folder = os.path.join(predictions_folder, name)
+ if not os.path.exists(use_predictions_folder):
+ use_predictions_folder = predictions_folder
+
for sample in tqdm(dataset):
stem = sample[0]
image = sample[1]
gt = sample[2]
- pred_fullpath = os.path.join(predictions_folder, stem + ".hdf5")
+ pred_fullpath = os.path.join(use_predictions_folder, stem + ".hdf5")
with h5py.File(pred_fullpath, "r") as f:
pred = f["array"][:]
pred = torch.from_numpy(pred)
@@ -288,7 +292,7 @@ def run(
overlay_image = _sample_analysis(
image, pred, gt, threshold=threshold, overlay=True
)
- fullpath = os.path.join(overlayed_folder, f"{stem}.png")
+ fullpath = os.path.join(overlayed_folder, name, f"{stem}.png")
tqdm.write(f"Saving {fullpath}...")
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
overlay_image.save(fullpath)
@@ -409,7 +413,7 @@ def compare_annotators(baseline, other, name, output_folder,
image, pred, gt, threshold=0.5, overlay=True
)
fullpath = os.path.join(overlayed_folder, "second-annotator",
- f"{stem}.png")
+ name, f"{stem}.png")
tqdm.write(f"Saving {fullpath}...")
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
overlay_image.save(fullpath)
diff --git a/bob/ip/binseg/engine/predictor.py b/bob/ip/binseg/engine/predictor.py
index 89d24d08..95fc1b76 100644
--- a/bob/ip/binseg/engine/predictor.py
+++ b/bob/ip/binseg/engine/predictor.py
@@ -101,7 +101,7 @@ def _save_overlayed_png(stem, image, prob, output_folder):
_save_image(stem, ".png", overlayed_image(image, prob), output_folder)
-def run(model, data_loader, device, output_folder, overlayed_folder):
+def run(model, data_loader, name, device, output_folder, overlayed_folder):
"""
Runs inference on input data, outputs HDF5 files with predictions
@@ -112,6 +112,10 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
data_loader : py:class:`torch.torch.utils.data.DataLoader`
+ name : str
+ the local name of this dataset (e.g. ``train``, or ``test``), to be
+ used when saving measures files.
+
device : str
device to use ``cpu`` or ``cuda:0``
@@ -138,9 +142,14 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
times = []
len_samples = []
- for samples in tqdm(
- data_loader, desc="batches", leave=False, disable=None,
- ):
+ output_folder = os.path.join(output_folder, name)
+ overlayed_folder = (
+ os.path.join(overlayed_folder, name)
+ if overlayed_folder is not None
+ else overlayed_folder
+ )
+
+ for samples in tqdm(data_loader, desc="batches", leave=False, disable=None):
names = samples[0]
images = samples[1].to(
diff --git a/bob/ip/binseg/script/evaluate.py b/bob/ip/binseg/script/evaluate.py
index 1cfc2bf1..cbd11aae 100644
--- a/bob/ip/binseg/script/evaluate.py
+++ b/bob/ip/binseg/script/evaluate.py
@@ -45,38 +45,6 @@ def _validate_threshold(t, dataset):
return t
-def _get_folder(folder, name):
- """Guesses the prediction folder to use based on the dataset name
-
- This function will look for ``folder/name`` if it exists, and
- return this. Otherwise defaults to ``folder``.
-
-
- Parameters
- ==========
-
- folder : str
- Path to the root of the predictions folder
-
- name : str
- The name of the dataset for which we are trying to find the predictions
- folder
-
-
- Returns
- =======
-
- path : str
- The best path to use as the root of the predictions folder for this
- dataset.
-
- """
- candidate = os.path.join(folder, name)
- if os.path.exists(candidate):
- return candidate
- return folder
-
-
@click.command(
entry_point_group="bob.ip.binseg.config",
cls=ConfigCommand,
@@ -208,13 +176,14 @@ def evaluate(
# first run evaluation for reference dataset, do not save overlays
logger.info(f"Evaluating threshold on '{threshold}' set")
threshold = run(
- dataset[threshold],
- threshold,
- _get_folder(predictions_folder, threshold),
- steps=steps,
+ dataset[threshold], threshold, predictions_folder, steps=steps
)
logger.info(f"Set --threshold={threshold:.5f}")
+ # clean-up the overlayed path
+ if overlayed is not None:
+ overlayed = overlayed.strip()
+
# now run with the
for k, v in dataset.items():
if k.startswith("_"):
@@ -224,7 +193,7 @@ def evaluate(
run(
v,
k,
- _get_folder(predictions_folder, k),
+ predictions_folder,
output_folder,
overlayed,
threshold,
diff --git a/bob/ip/binseg/script/predict.py b/bob/ip/binseg/script/predict.py
index 06a2edbf..f87e8bcb 100644
--- a/bob/ip/binseg/script/predict.py
+++ b/bob/ip/binseg/script/predict.py
@@ -145,8 +145,4 @@ def predict(output_folder, model, dataset, batch_size, device, weight,
shuffle=False,
pin_memory=torch.cuda.is_available(),
)
- # this avoids collisions if we have, e.g., multi-resolution versions
- # of the same dataset being evaluated, or datasets for which filenames
- # may match.
- use_output_folder = os.path.join(output_folder, k)
- run(model, data_loader, device, use_output_folder, overlayed)
+ run(model, data_loader, k, device, output_folder, overlayed)
diff --git a/bob/ip/binseg/test/test_cli.py b/bob/ip/binseg/test/test_cli.py
index 2b188270..decd28fd 100644
--- a/bob/ip/binseg/test/test_cli.py
+++ b/bob/ip/binseg/test/test_cli.py
@@ -117,20 +117,30 @@ def _check_experiment_stare(overlay):
# check predictions are there
predict_folder = os.path.join(output_folder, "predictions")
- basedir = os.path.join(predict_folder, "stare-images")
- assert os.path.exists(basedir)
- nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 20)
+ traindir = os.path.join(predict_folder, "train", "stare-images")
+ assert os.path.exists(traindir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(traindir), "*.hdf5")), 10)
+ testdir = os.path.join(predict_folder, "test", "stare-images")
+ assert os.path.exists(testdir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(testdir), "*.hdf5")), 10)
overlay_folder = os.path.join(output_folder, "overlayed", "predictions")
- basedir = os.path.join(overlay_folder, "stare-images")
+ traindir = os.path.join(overlay_folder, "train", "stare-images")
+ testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
# check overlayed images are there (since we requested them)
- assert os.path.exists(basedir)
+ assert os.path.exists(traindir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+ )
+ # check overlayed images are there (since we requested them)
+ assert os.path.exists(testdir)
nose.tools.eq_(
- len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+ len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
- assert not os.path.exists(basedir)
+ assert not os.path.exists(traindir)
+ assert not os.path.exists(testdir)
# check evaluation outputs
eval_folder = os.path.join(output_folder, "analysis")
@@ -144,29 +154,41 @@ def _check_experiment_stare(overlay):
)
overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
- basedir = os.path.join(overlay_folder, "stare-images")
+ traindir = os.path.join(overlay_folder, "train", "stare-images")
+ testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
# check overlayed images are there (since we requested them)
- assert os.path.exists(basedir)
+ assert os.path.exists(traindir)
nose.tools.eq_(
- len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+ len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+ )
+ assert os.path.exists(testdir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
- assert not os.path.exists(basedir)
+ assert not os.path.exists(traindir)
+ assert not os.path.exists(testdir)
# check overlayed images from first-to-second annotator comparisons
# are there (since we requested them)
overlay_folder = os.path.join(
output_folder, "overlayed", "analysis", "second-annotator"
)
- basedir = os.path.join(overlay_folder, "stare-images")
+ traindir = os.path.join(overlay_folder, "train", "stare-images")
+ testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
- assert os.path.exists(basedir)
+ assert os.path.exists(traindir)
nose.tools.eq_(
- len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
+ len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
+ )
+ assert os.path.exists(testdir)
+ nose.tools.eq_(
+ len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
- assert not os.path.exists(basedir)
+ assert not os.path.exists(traindir)
+ assert not os.path.exists(testdir)
# check outcomes of the comparison phase
assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))
@@ -388,7 +410,7 @@ def _check_evaluate(runner):
)
# check overlayed images are there (since we requested them)
- basedir = os.path.join(overlay_folder, "stare-images")
+ basedir = os.path.join(overlay_folder, "test", "stare-images")
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
--
GitLab