Skip to content
Snippets Groups Projects
Commit 1462b43b authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

[engine,script] Simplify implementation of subset folder structure; Fix unit tests

parent 9ebfcb57
No related branches found
No related tags found
No related merge requests found
Pipeline #40010 failed
......@@ -270,11 +270,15 @@ def run(
# Collect overall measures
data = {}
use_predictions_folder = os.path.join(predictions_folder, name)
if not os.path.exists(use_predictions_folder):
use_predictions_folder = predictions_folder
for sample in tqdm(dataset):
stem = sample[0]
image = sample[1]
gt = sample[2]
pred_fullpath = os.path.join(predictions_folder, stem + ".hdf5")
pred_fullpath = os.path.join(use_predictions_folder, stem + ".hdf5")
with h5py.File(pred_fullpath, "r") as f:
pred = f["array"][:]
pred = torch.from_numpy(pred)
......@@ -288,7 +292,7 @@ def run(
overlay_image = _sample_analysis(
image, pred, gt, threshold=threshold, overlay=True
)
fullpath = os.path.join(overlayed_folder, f"{stem}.png")
fullpath = os.path.join(overlayed_folder, name, f"{stem}.png")
tqdm.write(f"Saving {fullpath}...")
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
overlay_image.save(fullpath)
......@@ -409,7 +413,7 @@ def compare_annotators(baseline, other, name, output_folder,
image, pred, gt, threshold=0.5, overlay=True
)
fullpath = os.path.join(overlayed_folder, "second-annotator",
f"{stem}.png")
name, f"{stem}.png")
tqdm.write(f"Saving {fullpath}...")
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
overlay_image.save(fullpath)
......
......@@ -101,7 +101,7 @@ def _save_overlayed_png(stem, image, prob, output_folder):
_save_image(stem, ".png", overlayed_image(image, prob), output_folder)
def run(model, data_loader, device, output_folder, overlayed_folder):
def run(model, data_loader, name, device, output_folder, overlayed_folder):
"""
Runs inference on input data, outputs HDF5 files with predictions
......@@ -112,6 +112,10 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
data_loader : py:class:`torch.torch.utils.data.DataLoader`
name : str
the local name of this dataset (e.g. ``train``, or ``test``), to be
used when saving measures files.
device : str
device to use ``cpu`` or ``cuda:0``
......@@ -138,9 +142,14 @@ def run(model, data_loader, device, output_folder, overlayed_folder):
times = []
len_samples = []
for samples in tqdm(
data_loader, desc="batches", leave=False, disable=None,
):
output_folder = os.path.join(output_folder, name)
overlayed_folder = (
os.path.join(overlayed_folder, name)
if overlayed_folder is not None
else overlayed_folder
)
for samples in tqdm(data_loader, desc="batches", leave=False, disable=None):
names = samples[0]
images = samples[1].to(
......
......@@ -45,38 +45,6 @@ def _validate_threshold(t, dataset):
return t
def _get_folder(folder, name):
"""Guesses the prediction folder to use based on the dataset name
This function will look for ``folder/name`` if it exists, and
return this. Otherwise defaults to ``folder``.
Parameters
==========
folder : str
Path to the root of the predictions folder
name : str
The name of the dataset for which we are trying to find the predictions
folder
Returns
=======
path : str
The best path to use as the root of the predictions folder for this
dataset.
"""
candidate = os.path.join(folder, name)
if os.path.exists(candidate):
return candidate
return folder
@click.command(
entry_point_group="bob.ip.binseg.config",
cls=ConfigCommand,
......@@ -208,13 +176,14 @@ def evaluate(
# first run evaluation for reference dataset, do not save overlays
logger.info(f"Evaluating threshold on '{threshold}' set")
threshold = run(
dataset[threshold],
threshold,
_get_folder(predictions_folder, threshold),
steps=steps,
dataset[threshold], threshold, predictions_folder, steps=steps
)
logger.info(f"Set --threshold={threshold:.5f}")
# clean-up the overlayed path
if overlayed is not None:
overlayed = overlayed.strip()
# now run with the
for k, v in dataset.items():
if k.startswith("_"):
......@@ -224,7 +193,7 @@ def evaluate(
run(
v,
k,
_get_folder(predictions_folder, k),
predictions_folder,
output_folder,
overlayed,
threshold,
......
......@@ -145,8 +145,4 @@ def predict(output_folder, model, dataset, batch_size, device, weight,
shuffle=False,
pin_memory=torch.cuda.is_available(),
)
# this avoids collisions if we have, e.g., multi-resolution versions
# of the same dataset being evaluated, or datasets for which filenames
# may match.
use_output_folder = os.path.join(output_folder, k)
run(model, data_loader, device, use_output_folder, overlayed)
run(model, data_loader, k, device, output_folder, overlayed)
......@@ -117,20 +117,30 @@ def _check_experiment_stare(overlay):
# check predictions are there
predict_folder = os.path.join(output_folder, "predictions")
basedir = os.path.join(predict_folder, "stare-images")
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 20)
traindir = os.path.join(predict_folder, "train", "stare-images")
assert os.path.exists(traindir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(traindir), "*.hdf5")), 10)
testdir = os.path.join(predict_folder, "test", "stare-images")
assert os.path.exists(testdir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(testdir), "*.hdf5")), 10)
overlay_folder = os.path.join(output_folder, "overlayed", "predictions")
basedir = os.path.join(overlay_folder, "stare-images")
traindir = os.path.join(overlay_folder, "train", "stare-images")
testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
# check overlayed images are there (since we requested them)
assert os.path.exists(basedir)
assert os.path.exists(traindir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
)
# check overlayed images are there (since we requested them)
assert os.path.exists(testdir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
assert not os.path.exists(basedir)
assert not os.path.exists(traindir)
assert not os.path.exists(testdir)
# check evaluation outputs
eval_folder = os.path.join(output_folder, "analysis")
......@@ -144,29 +154,41 @@ def _check_experiment_stare(overlay):
)
overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
basedir = os.path.join(overlay_folder, "stare-images")
traindir = os.path.join(overlay_folder, "train", "stare-images")
testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
# check overlayed images are there (since we requested them)
assert os.path.exists(basedir)
assert os.path.exists(traindir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
)
assert os.path.exists(testdir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
assert not os.path.exists(basedir)
assert not os.path.exists(traindir)
assert not os.path.exists(testdir)
# check overlayed images from first-to-second annotator comparisons
# are there (since we requested them)
overlay_folder = os.path.join(
output_folder, "overlayed", "analysis", "second-annotator"
)
basedir = os.path.join(overlay_folder, "stare-images")
traindir = os.path.join(overlay_folder, "train", "stare-images")
testdir = os.path.join(overlay_folder, "test", "stare-images")
if overlay:
assert os.path.exists(basedir)
assert os.path.exists(traindir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(basedir), "*.png")), 20
len(fnmatch.filter(os.listdir(traindir), "*.png")), 10
)
assert os.path.exists(testdir)
nose.tools.eq_(
len(fnmatch.filter(os.listdir(testdir), "*.png")), 10
)
else:
assert not os.path.exists(basedir)
assert not os.path.exists(traindir)
assert not os.path.exists(testdir)
# check outcomes of the comparison phase
assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))
......@@ -388,7 +410,7 @@ def _check_evaluate(runner):
)
# check overlayed images are there (since we requested them)
basedir = os.path.join(overlay_folder, "stare-images")
basedir = os.path.join(overlay_folder, "test", "stare-images")
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment