[scripts] Streamline saliency map generation and analysis

src/ptbench/data/datamodule.py

@@ -592,7 +592,8 @@ class ConcatDataModule(lightning.LightningDataModule):
             ] = multiprocessing.get_context("spawn")
 
         # keep workers hanging around if we have multiple
-        self._dataloader_multiproc["persistent_workers"] = True
+        if value >= 0:
+            self._dataloader_multiproc["persistent_workers"] = True
 
     @property
     def model_transforms(self) -> list[Transform] | None:

src/ptbench/engine/saliency/generator.py

@@ -101,7 +101,7 @@ def run(
     model: lightning.pytorch.LightningModule,
     datamodule: lightning.pytorch.LightningDataModule,
     device_manager: DeviceManager,
-    saliency_map_algorithms: typing.Sequence[SaliencyMapAlgorithm],
+    saliency_map_algorithm: SaliencyMapAlgorithm,
     target_class: typing.Literal["highest", "all"],
     positive_only: bool,
     output_folder: pathlib.Path,
@@ -119,8 +119,8 @@ def run(
         An internal device representation, to be used for training and
         validation.  This representation can be converted into a pytorch device
         or a torch lightning accelerator setup.
-    saliency_map_algorithms
-        The algorithms for saliency map estimation to use.
+    saliency_map_algorithm
+        The algorithm to use for saliency map estimation.
     target_class
         (Use only with multi-label models) Which class to target for CAM
         calculation. Can be either set to "all" or "highest". "highest" is
@@ -140,7 +140,7 @@ def run(
     from ...models.pasa import Pasa
 
     if isinstance(model, Pasa):
-        if "fullgrad" in saliency_map_algorithms:
+        if saliency_map_algorithm == "fullgrad":
             raise ValueError(
                 "Fullgrad saliency map algorithm is not supported for the "
                 "Pasa model."
@@ -160,71 +160,63 @@ def run(
     model = model.to(device)
     model.eval()
 
-    for algo_type in saliency_map_algorithms:
-        saliency_map_callable = _create_saliency_map_callable(
-            algo_type,
-            model,
-            target_layers,  # type: ignore
-            use_cuda,
+    saliency_map_callable = _create_saliency_map_callable(
+        saliency_map_algorithm,
+        model,
+        target_layers,  # type: ignore
+        use_cuda,
+    )
+
+    for k, v in datamodule.predict_dataloader().items():
+        logger.info(
+            f"Generating saliency maps for dataset `{k}` via `{saliency_map_algorithm}`..."
         )
 
-        for k, v in datamodule.predict_dataloader().items():
-            logger.info(
-                f"Generating saliency maps for dataset `{k}` via `{algo_type}`..."
+        for sample in tqdm.tqdm(v, desc="samples", leave=False, disable=None):
+            name = sample[1]["name"][0]
+            label = sample[1]["label"].item()
+            image = sample[0].to(
+                device=device, non_blocking=torch.cuda.is_available()
             )
 
-            for sample in tqdm.tqdm(
-                v, desc="samples", leave=False, disable=None
-            ):
-                name = sample[1]["name"][0]
-                label = sample[1]["label"].item()
-                image = sample[0].to(
-                    device=device, non_blocking=torch.cuda.is_available()
-                )
-
-                # in binary classification systems, negative labels may be skipped
-                if positive_only and (model.num_classes == 1) and (label == 0):
-                    continue
-
-                # chooses target outputs to generate saliency maps for
-                if model.num_classes > 1:
-                    if target_class == "all":
-                        # just blindly generate saliency maps for all outputs
-                        # - make one directory for every target output and lay
-                        # images there like in the original dataset.
-                        for output_num in range(model.num_classes):
-                            use_folder = (
-                                output_folder / algo_type / str(output_num)
-                            )
-                            saliency_map = saliency_map_callable(
-                                input_tensor=image,
-                                targets=[ClassifierOutputTarget(output_num)],  # type: ignore
-                            )
-                            _save_saliency_map(use_folder, name, saliency_map)  # type: ignore
-
-                    else:
-                        # pytorch-grad-cam will figure out the output with the
-                        # highest value and produce a saliency map for it - we
-                        # will save it to disk.
-                        use_folder = (
-                            output_folder / algo_type / "highest-output"
-                        )
+            # in binary classification systems, negative labels may be skipped
+            if positive_only and (model.num_classes == 1) and (label == 0):
+                continue
+
+            # chooses target outputs to generate saliency maps for
+            if model.num_classes > 1:
+                if target_class == "all":
+                    # just blindly generate saliency maps for all outputs
+                    # - make one directory for every target output and lay
+                    # images there like in the original dataset.
+                    for output_num in range(model.num_classes):
+                        use_folder = output_folder / str(output_num)
                         saliency_map = saliency_map_callable(
                             input_tensor=image,
-                            # setting `targets=None` will set target to the
-                            # maximum output index using
-                            # ClassifierOutputTarget(max_output_index)
-                            targets=None,  # type: ignore
+                            targets=[ClassifierOutputTarget(output_num)],  # type: ignore
                         )
                         _save_saliency_map(use_folder, name, saliency_map)  # type: ignore
+
                 else:
-                    # binary classification model with a single output - just
-                    # lay all cams uniformily like the original dataset
-                    use_folder = output_folder / algo_type
+                    # pytorch-grad-cam will figure out the output with the
+                    # highest value and produce a saliency map for it - we
+                    # will save it to disk.
+                    use_folder = output_folder / "highest-output"
                     saliency_map = saliency_map_callable(
                         input_tensor=image,
-                        targets=[
-                            ClassifierOutputTarget(0),  # type: ignore
-                        ],
+                        # setting `targets=None` will set target to the
+                        # maximum output index using
+                        # ClassifierOutputTarget(max_output_index)
+                        targets=None,  # type: ignore
                     )
                     _save_saliency_map(use_folder, name, saliency_map)  # type: ignore
+            else:
+                # binary classification model with a single output - just
+                # lay all cams uniformily like the original dataset
+                saliency_map = saliency_map_callable(
+                    input_tensor=image,
+                    targets=[
+                        ClassifierOutputTarget(0),  # type: ignore
+                    ],
+                )
+                _save_saliency_map(output_folder, name, saliency_map)  # type: ignore

src/ptbench/engine/saliency/interpretability.py

@@ -269,7 +269,7 @@ def _compute_proportional_energy(
 def _process_sample(
     gt_bboxes: BoundingBoxes,
     saliency_map: numpy.typing.NDArray[numpy.double],
-) -> tuple[float, float, float, float, BoundingBox]:
+) -> tuple[float, float, float, float, tuple[int, int, int, int]]:
     """Calculates the metrics for a single sample.
 
     Parameters
@@ -310,7 +310,12 @@ def _process_sample(
         ioda,
         _compute_proportional_energy(saliency_map, binary_mask),
         _compute_avg_saliency_focus(saliency_map, binary_mask),
-        detected_box,
+        (
+            detected_box.xmin,
+            detected_box.ymin,
+            detected_box.width,
+            detected_box.height,
+        ),
     )
 
 
@@ -396,7 +401,7 @@ def run(
                     name,
                     label,
                     *_process_sample(
-                        bboxes,
+                        bboxes[0],
                         numpy.load(
                             input_folder
                             / pathlib.Path(name).with_suffix(".npy")

src/ptbench/scripts/experiment.py

@@ -111,11 +111,6 @@ def experiment(
 
     logger.info("Started predicting")
 
-    from ..utils.checkpointer import get_checkpoint_to_run_inference
-
-    model_file = get_checkpoint_to_run_inference(train_output_folder)
-    logger.info(f"Found `{str(model_file)}`. Continuing...")
-
     from .predict import predict
 
     predictions_output = output_folder / "predictions.json"
@@ -126,7 +121,7 @@ def experiment(
         model=model,
         datamodule=datamodule,
         device=device,
-        weight=model_file,
+        weight=train_output_folder,
         batch_size=batch_size,
         parallel=parallel,
     )

src/ptbench/scripts/predict.py

@@ -23,13 +23,13 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 
    .. code:: sh
 
-      ptbench predict -vv pasa montgomery --weight=path/to/model.ckpt --output=path/to/predictions.json
+      ptbench predict -vv pasa montgomery --weight=path/to/model-at-lowest-validation-loss.ckpt --output=path/to/predictions.json
 
 2. Enables multi-processing data loading with 6 processes:
 
    .. code:: sh
 
-      ptbench predict -vv pasa montgomery --parallel=6 --weight=path/to/model.ckpt --output=path/to/predictions.json
+      ptbench predict -vv pasa montgomery --parallel=6 --weight=path/to/model-at-lowest-validation-loss.ckpt --output=path/to/predictions.json
 
 """,
 )
@@ -88,10 +88,18 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
     "--weight",
     "-w",
     help="""Path or URL to pretrained model file (`.ckpt` extension),
-    corresponding to the architecture set with `--model`.""",
+    corresponding to the architecture set with `--model`.  Optionally, you may
+    also pass a directory containing the result of a training session, in which
+    case either the best (lowest validation) or latest model will be loaded.""",
     required=True,
     cls=ResourceOption,
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=True,
+        readable=True,
+        path_type=pathlib.Path,
+    ),
 )
 @click.option(
     "--parallel",
@@ -125,6 +133,7 @@ def predict(
 
     from ..engine.device import DeviceManager
     from ..engine.predictor import run
+    from ..utils.checkpointer import get_checkpoint_to_run_inference
 
     datamodule.set_chunk_size(batch_size, 1)
     datamodule.parallel = parallel
@@ -133,6 +142,9 @@ def predict(
     datamodule.prepare_data()
     datamodule.setup(stage="predict")
 
+    if weight.is_dir():
+        weight = get_checkpoint_to_run_inference(weight)
+
     logger.info(f"Loading checkpoint from `{weight}`...")
     model = type(model).load_from_checkpoint(weight, strict=False)
 

src/ptbench/scripts/saliency/completeness.py

@@ -25,7 +25,7 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 
    .. code:: sh
 
-      ptbench saliency completeness -vv pasa tbx11k-v1-healthy-vs-atb --device="cuda" --weight=path/to/model-at-lowest-validation-loss.ckpt --output-folder=path/to/completeness-scores/
+      ptbench saliency completeness -vv pasa tbx11k-v1-healthy-vs-atb --device="cuda" --weight=path/to/model-at-lowest-validation-loss.ckpt --output-json=path/to/completeness-scores.json
 
 """,
 )
@@ -49,18 +49,17 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
     cls=ResourceOption,
 )
 @click.option(
-    "--output-folder",
+    "--output-json",
     "-o",
-    help="Path where to store saliency maps (created if does not exist)",
+    help="""Path where to store the output JSON file containing all
+    measures.""",
     required=True,
     type=click.Path(
-        exists=False,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
+        file_okay=True,
+        dir_okay=False,
         path_type=pathlib.Path,
     ),
-    default="saliency-maps",
+    default="saliency-interpretability.json",
     cls=ResourceOption,
 )
 @click.option(
@@ -85,10 +84,18 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
     "--weight",
     "-w",
     help="""Path or URL to pretrained model file (`.ckpt` extension),
-    corresponding to the architecture set with `--model`.""",
+    corresponding to the architecture set with `--model`.  Optionally, you may
+    also pass a directory containing the result of a training session, in which
+    case either the best (lowest validation) or latest model will be loaded.""",
     required=True,
     cls=ResourceOption,
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=True,
+        readable=True,
+        path_type=pathlib.Path,
+    ),
 )
 @click.option(
     "--parallel",
@@ -108,13 +115,11 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 @click.option(
     "--saliency-map-algorithm",
     "-s",
-    help="""Saliency map algorithm(s) to be used. Can be called multiple times
-    with different techniques.""",
+    help="""Saliency map algorithm to be used.""",
     type=click.Choice(
         typing.get_args(SaliencyMapAlgorithm), case_sensitive=False
     ),
-    multiple=True,
-    default=["gradcam"],
+    default="gradcam",
     show_default=True,
     cls=ResourceOption,
 )
@@ -157,7 +162,7 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 def completeness(
     model,
     datamodule,
-    output_folder,
+    output_json,
     device,
     cache_samples,
     weight,
@@ -171,7 +176,7 @@ def completeness(
     """Evaluates saliency map algorithm completeness using RemOve And Debias
     (ROAD).
 
-    For each selected saliency map algorithm, evaluates the completeness of
+    For the selected saliency map algorithm, evaluates the completeness of
     explanations using the RemOve And Debias (ROAD) algorithm. The ROAD
     algorithm was first described at [ROAD-2022]_. It estimates explainability
     (in the completeness sense) of saliency mapping algorithms by substituting
@@ -185,12 +190,9 @@ def completeness(
 
     This program outputs a JSON file containing the ROAD evaluations (using
     most-relevant-first, or MoRF, and least-relevant-first, or LeRF for each
-    sample in the datamodule, and per saliency-mapping algorithm.  Each
-    saliency-mapping algorithm yields a single JSON file with the target
-    algorithm name on the ``output-folder``. Values for MoRF and LeRF represent
-    averages by removing 20, 40, 60 and 80% of most or least relevant pixels
-    respectively from the image, and averaging results for all these
-    percentiles.
+    sample in the datamodule. Values for MoRF and LeRF represent averages by
+    removing 20, 40, 60 and 80% of most or least relevant pixels respectively
+    from the image, and averaging results for all these percentiles.
 
     .. note::
 
@@ -201,9 +203,7 @@ def completeness(
 
     from ...engine.device import DeviceManager
     from ...engine.saliency.completeness import run
-
-    logger.info(f"Output folder: {output_folder}")
-    output_folder.mkdir(parents=True, exist_ok=True)
+    from ...utils.checkpointer import get_checkpoint_to_run_inference
 
     if device in ("cuda", "mps") and (parallel == 0 or parallel > 1):
         raise RuntimeError(
@@ -226,27 +226,29 @@ def completeness(
     datamodule.prepare_data()
     datamodule.setup(stage="predict")
 
-    logger.info(f"Loading checkpoint from `{weight}`...")
-    model = model.load_from_checkpoint(weight, strict=False)
-
-    for algo in saliency_map_algorithm:
-        logger.info(
-            f"Evaluating RemOve And Debias (ROAD) average scores for "
-            f"algorithm `{algo}` with percentiles "
-            f"`{', '.join([str(k) for k in percentile])}`..."
-        )
-        results = run(
-            model=model,
-            datamodule=datamodule,
-            device_manager=device_manager,
-            saliency_map_algorithm=algo,
-            target_class=target_class,
-            positive_only=positive_only,
-            percentiles=percentile,
-            parallel=parallel,
-        )
+    if weight.is_dir():
+        weight = get_checkpoint_to_run_inference(weight)
 
-        output_json = output_folder / (algo + ".json")
-        with output_json.open("w") as f:
-            logger.info(f"Saving output file to `{str(output_json)}`...")
-            json.dump(results, f, indent=2)
+    logger.info(f"Loading checkpoint from `{weight}`...")
+    model = type(model).load_from_checkpoint(weight, strict=False)
+
+    logger.info(
+        f"Evaluating RemOve And Debias (ROAD) average scores for "
+        f"algorithm `{saliency_map_algorithm}` with percentiles "
+        f"`{', '.join([str(k) for k in percentile])}`..."
+    )
+    results = run(
+        model=model,
+        datamodule=datamodule,
+        device_manager=device_manager,
+        saliency_map_algorithm=saliency_map_algorithm,
+        target_class=target_class,
+        positive_only=positive_only,
+        percentiles=percentile,
+        parallel=parallel,
+    )
+
+    output_json.parent.mkdir(parents=True, exist_ok=True)
+    with output_json.open("w") as f:
+        logger.info(f"Saving output file to `{str(output_json)}`...")
+        json.dump(results, f, indent=2)

src/ptbench/scripts/saliency/generate.py

@@ -87,10 +87,18 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
     "--weight",
     "-w",
     help="""Path or URL to pretrained model file (`.ckpt` extension),
-    corresponding to the architecture set with `--model`.""",
+    corresponding to the architecture set with `--model`.  Optionally, you may
+    also pass a directory containing the result of a training session, in which
+    case either the best (lowest validation) or latest model will be loaded.""",
     required=True,
     cls=ResourceOption,
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=True,
+        readable=True,
+        path_type=pathlib.Path,
+    ),
 )
 @click.option(
     "--parallel",
@@ -108,13 +116,11 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 @click.option(
     "--saliency-map-algorithm",
     "-s",
-    help="""Saliency map algorithm(s) to be used. Can be called multiple times
-    with different techniques.""",
+    help="""Saliency map algorithm to be used.""",
     type=click.Choice(
         typing.get_args(SaliencyMapAlgorithm), case_sensitive=False
     ),
-    multiple=True,
-    default=["gradcam"],
+    default="gradcam",
     show_default=True,
     cls=ResourceOption,
 )
@@ -165,6 +171,7 @@ def generate(
 
     from ...engine.device import DeviceManager
     from ...engine.saliency.generator import run
+    from ...utils.checkpointer import get_checkpoint_to_run_inference
 
     logger.info(f"Output folder: {output_folder}")
     output_folder.mkdir(parents=True, exist_ok=True)
@@ -181,14 +188,17 @@ def generate(
     datamodule.prepare_data()
     datamodule.setup(stage="predict")
 
+    if weight.is_dir():
+        weight = get_checkpoint_to_run_inference(weight)
+
     logger.info(f"Loading checkpoint from `{weight}`...")
-    model = model.load_from_checkpoint(weight, strict=False)
+    model = type(model).load_from_checkpoint(weight, strict=False)
 
     run(
         model=model,
         datamodule=datamodule,
         device_manager=device_manager,
-        saliency_map_algorithms=saliency_map_algorithm,
+        saliency_map_algorithm=saliency_map_algorithm,
         target_class=target_class,
         positive_only=positive_only,
         output_folder=output_folder,

src/ptbench/scripts/saliency/interpretability.py

@@ -23,7 +23,7 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
 
    .. code:: sh
 
-      ptbench saliency interpretability -vv tbx11k-v1-healthy-vs-atb --input-folder=parent_folder/gradcam/ --output-json=parent_folder/gradcam/tbx11k-v1-interp.json
+      ptbench saliency interpretability -vv tbx11k-v1-healthy-vs-atb --input-folder=parent-folder/saliencies/ --output-json=path/to/interpretability-scores.json
 
 """,
 )

src/ptbench/utils/checkpointer.py

@@ -139,7 +139,7 @@ def get_checkpoint_to_run_inference(
     """
 
     try:
-        _get_checkpoint_from_alias(path, "best")
+        return _get_checkpoint_from_alias(path, "best")
     except FileNotFoundError:
         logger.error(
             "Did not find lowest-validation-loss model to run inference "

tests/test_cam_utils.py

@@ -3,19 +3,17 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 """Tests for the cam_utils script."""
 
-import cv2
 import numpy as np
-import pandas as pd
 import pytest
 
-from ptbench.utils.cam_utils import (
-    _calculate_stats_over_dataset,
-    calculate_metrics_avg_for_every_class,
-    draw_boxes_on_image,
-    draw_largest_component_bbox_on_image,
-    show_cam_on_image,
-    visualize_road_scores,
-)
+# from ptbench.utils.cam_utils import (
+#     _calculate_stats_over_dataset,
+#     calculate_metrics_avg_for_every_class,
+#     draw_boxes_on_image,
+#     draw_largest_component_bbox_on_image,
+#     show_cam_on_image,
+#     visualize_road_scores,
+# )
 
 
 def test_calculate_stats_over_dataset(datadir):