From fcb5475675d4353c5eb46586bcd357ba4a5fb551 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Tue, 7 Apr 2020 13:14:31 +0200
Subject: [PATCH] [predictor] Multiple fixes so it works confortably
---
bob/ip/binseg/engine/predictor.py | 22 ++++++++++++++++------
bob/ip/binseg/script/predict.py | 20 +++++++++++++-------
doc/evaluation.rst | 28 +++++++++++++++++++---------
3 files changed, 48 insertions(+), 22 deletions(-)
diff --git a/bob/ip/binseg/engine/predictor.py b/bob/ip/binseg/engine/predictor.py
index 016ce0bc..093c6c67 100644
--- a/bob/ip/binseg/engine/predictor.py
+++ b/bob/ip/binseg/engine/predictor.py
@@ -4,10 +4,13 @@
import os
import time
import datetime
-import numpy as np
+
+import numpy
import torch
from tqdm import tqdm
+import bob.io.base
+
import logging
logger = logging.getLogger(__name__)
@@ -35,9 +38,11 @@ def save_hdf5(predictions, names, output_folder):
img = predictions.cpu().data[j].squeeze(0).numpy()
filename = "{}.hdf5".format(names[j].split(".")[0])
fullpath = os.path.join(output_folder, filename)
- logger.info(f"saving {filename}")
+ tqdm.write(f"Saving {fullpath}...")
fulldir = os.path.dirname(fullpath)
if not os.path.exists(fulldir):
+ tqdm.write(f"Creating directory {fulldir}...")
+ # protect against concurrent access - exist_ok=True
os.makedirs(fulldir, exist_ok=True)
bob.io.base.save(img, fullpath)
@@ -62,8 +67,13 @@ def run(model, data_loader, device, output_folder):
"""
logger.info("Start prediction")
- logger.info(f"Output folder: {output_folder}, Device: {device}")
- os.makedirs(output_folder, exist_ok=True)
+ logger.info(f"Output folder: {output_folder}")
+ logger.info(f"Device: {device}")
+
+ if not os.path.exists(output_folder):
+ logger.debug(f"Creating output directory '{output_folder}'...")
+ # protect against concurrent access - exist_ok=True
+ os.makedirs(output_folder, exist_ok=True)
model.eval().to(device)
# Sigmoid for probabilities
@@ -105,8 +115,8 @@ def run(model, data_loader, device, output_folder):
total_time = datetime.timedelta(seconds=int(time.time() - start_total_time))
logger.info(f"Total time: {total_time}")
- average_batch_time = np.mean(times)
+ average_batch_time = numpy.mean(times)
logger.info(f"Average batch time: {average_batch_time:g}s\n")
- average_image_time = np.sum(times * len_samples) / float(sum(len_samples))
+ average_image_time = numpy.sum(numpy.array(times) * len_samples) / float(sum(len_samples))
logger.info(f"Average image time: {average_image_time:g}s\n")
diff --git a/bob/ip/binseg/script/predict.py b/bob/ip/binseg/script/predict.py
index 8fe08c40..560a8a19 100644
--- a/bob/ip/binseg/script/predict.py
+++ b/bob/ip/binseg/script/predict.py
@@ -1,6 +1,8 @@
#!/usr/bin/env python
# coding=utf-8
+import os
+
import click
from click_plugins import with_plugins
@@ -15,6 +17,7 @@ from bob.extension.scripts.click_helper import (
)
from ..engine.predictor import run
+from ..utils.checkpointer import DetectronCheckpointer
import logging
logger = logging.getLogger(__name__)
@@ -27,7 +30,7 @@ logger = logging.getLogger(__name__)
\b
1. Runs prediction on an existing dataset configuration:
-
+\b
$ bob binseg predict -vv m2unet drive-test --weight=path/to/model_final.pth --output-path=path/to/predictions
\b
2. To run prediction on a folder with your own images, you must first
@@ -36,10 +39,10 @@ logger = logging.getLogger(__name__)
performance. To figure out such specifications, you must consult the
dataset configuration used for **training** the provided model. Once
you figured this out, do the following:
-
- $ bob binseg config copy image-folder myfolder.py
- # modify "myfolder.py" to include the base path and required transforms
- $ bob binseg predict -vv m2unet myfolder.py --weight=path/to/model_final.pth --output-path=path/to/predictions
+\b
+ $ bob binseg config copy folder-dataset-example mydataset.py
+ # modify "mydataset.py" to include the base path and required transforms
+ $ bob binseg predict -vv m2unet mydataset.py --weight=path/to/model_final.pth --output-path=path/to/predictions
""",
)
@click.option(
@@ -102,8 +105,11 @@ def predict(output_path, model, dataset, batch_size, device, weight, **kwargs):
)
# checkpointer, loads pre-fit model
- checkpointer = DetectronCheckpointer(model, save_dir=output_path,
+ weight_fullpath = os.path.abspath(weight)
+ weight_path = os.path.dirname(weight_fullpath)
+ weight_name = os.path.basename(weight_fullpath)
+ checkpointer = DetectronCheckpointer(model, save_dir=weight_path,
save_to_disk=False)
- checkpointer.load(weight)
+ checkpointer.load(weight_name)
run(model, data_loader, device, output_path)
diff --git a/doc/evaluation.rst b/doc/evaluation.rst
index 3b0f758d..0329dabd 100644
--- a/doc/evaluation.rst
+++ b/doc/evaluation.rst
@@ -6,13 +6,22 @@
Inference and Evaluation
==========================
+This guides explains how to run inference or a complete evaluation using
+command-line tools. Inference produces probability maps for input images,
+while evaluation will analyze such output against existing annotations and
+produce performance figures.
+
Inference
---------
You may use one of your trained models (or :ref:`one of ours
<bob.ip.binseg.models>` to run inference on existing datasets or your own
-dataset.
+dataset. In inference (or prediction) mode, we input data, the trained model,
+and output HDF5 files containing the prediction outputs for every input image.
+Each HDF5 file contains a single object with a 2-dimensional matrix of floating
+point numbers indicating the vessel probability (``[0.0,1.0]``) for each pixel
+in the input image.
Inference on an existing datasets
@@ -43,7 +52,6 @@ you need to instantiate one of:
Read the appropriate module documentation for details.
-
.. code-block:: bash
$ bob binseg config copy folder-dataset-example mydataset.py
@@ -53,16 +61,18 @@ Read the appropriate module documentation for details.
$ bob binseg predict -vv <model> -w <path/to/model.pth> ./mydataset.py
+Inference typically consumes less resources than training, but you may speed
+things up using ``--device='cuda:0'`` in case you have a GPU.
+
+
Evaluation
----------
-To evaluate trained models use our CLI interface. ``bob binseg evaluate``
-followed by the model and the dataset configuration, and the path to the
-pretrained model via the argument ``--weight``.
-
-Alternatively point to the output folder used during training via the
-``--output-path`` argument. The Checkpointer will load the model as indicated
-in the file: ``last_checkpoint``.
+In evaluation we input an **annotated** dataset and a pre-trained model to
+output a complete set of performance figures that can help analysis of model
+performance. Evaluation is done using ``bob binseg evaluate`` followed by the
+model and the dataset configuration, and the path to the pretrained model via
+the ``--weight`` argument.
Use ``bob binseg evaluate --help`` for more information.
--
GitLab