[tests.classification] Streamline testing

src/mednet/libs/classification/engine/saliency/viewer.py

@@ -221,7 +221,8 @@ def run(
 
     for dataset_name, dataset_loader in datamodule.predict_dataloader().items():
         logger.info(
-            f"Generating visualisations for samples at dataset `{dataset_name}`...",
+            f"Generating visualizations for samples (target = {target_label}) "
+            f"at dataset `{dataset_name}`..."
         )
 
         for sample in tqdm(

src/mednet/libs/classification/scripts/evaluate.py

@@ -148,7 +148,6 @@ def evaluate(
     )
     json_data = {k.replace("_", "-"): v for k, v in json_data.items()}
     evaluation_meta = evaluation_file.with_suffix(".meta.json")
-    logger.info(f"Saving evaluation metadata at `{str(evaluation_meta)}`...")
     save_json_with_backup(evaluation_meta, json_data)
 
     if threshold in predict_data:

src/mednet/libs/classification/scripts/saliency/generate.py

@@ -171,12 +171,37 @@ def generate(
 
     from mednet.libs.common.engine.device import DeviceManager
     from mednet.libs.common.scripts.predict import setup_datamodule
+    from mednet.libs.common.scripts.utils import (
+        execution_metadata,
+        save_json_with_backup,
+    )
     from mednet.libs.common.utils.checkpointer import (
         get_checkpoint_to_run_inference,
     )
 
     from ...engine.saliency.generator import run
 
+    # register metadata
+    json_data: dict[str, typing.Any] = execution_metadata()
+    json_data.update(
+        dict(
+            database_name=datamodule.database_name,
+            database_split=datamodule.split_name,
+            model_name=model.name,
+            output_folder=str(output_folder),
+            device=device,
+            cache_samples=cache_samples,
+            weight=str(weight),
+            parallel=parallel,
+            saliency_map_algorithm=saliency_map_algorithm,
+            target_class=target_class,
+            positive_only=positive_only,
+        ),
+    )
+    json_data = {k.replace("_", "-"): v for k, v in json_data.items()}
+    saliency_meta = output_folder / "saliency-generation.meta.json"
+    save_json_with_backup(saliency_meta, json_data)
+
     logger.info(f"Output folder: {output_folder}")
     output_folder.mkdir(parents=True, exist_ok=True)
 

src/mednet/libs/classification/scripts/saliency/view.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import pathlib
+import typing
 
 import click
 from clapper.click import ConfigCommand, ResourceOption, verbosity_option
@@ -97,19 +98,33 @@ def view(
     **_,
 ) -> None:  # numpydoc ignore=PR01
     """Generate heatmaps for input CXRs based on existing saliency maps."""
-    from ...engine.saliency.viewer import run
-
-    assert (
-        input_folder != output_folder
-    ), "Output folder must not be the same as the input folder."
+    from mednet.libs.common.scripts.utils import (
+        execution_metadata,
+        save_json_with_backup,
+    )
 
-    assert not str(output_folder).startswith(
-        str(input_folder),
-    ), "Output folder must not be a subdirectory of the input folder."
+    from ...engine.saliency.viewer import run
 
     logger.info(f"Output folder: {output_folder}")
     output_folder.mkdir(parents=True, exist_ok=True)
 
+    # register metadata
+    json_data: dict[str, typing.Any] = execution_metadata()
+    json_data.update(
+        dict(
+            database_name=datamodule.database_name,
+            database_split=datamodule.split_name,
+            model_name=model.name,
+            input_folder=str(input_folder),
+            output_folder=str(output_folder),
+            show_groundtruth=show_groundtruth,
+            threshold=threshold,
+        ),
+    )
+    json_data = {k.replace("_", "-"): v for k, v in json_data.items()}
+    saliency_meta = output_folder / "saliency-view.meta.json"
+    save_json_with_backup(saliency_meta, json_data)
+
     datamodule.drop_incomplete_batch = False
     # datamodule.cache_samples = cache_samples
     # datamodule.parallel = parallel

tests/classification/test_cli.py

@@ -219,6 +219,7 @@ def test_train_pasa_montgomery(session_tmp_path):
         assert (output_folder / "meta.json").exists()
 
         keywords = {
+            r"^Writing run metadata at .*$": 1,
             r"^Loading dataset:`train` without caching. Trade-off: CPU RAM usage: less | Disk I/O: more.$": 1,
             r"^Loading dataset:`validation` without caching. Trade-off: CPU RAM usage: less | Disk I/O: more.$": 1,
             r"^Applying train/valid loss balancing...$": 1,
@@ -271,6 +272,7 @@ def test_predict_pasa_montgomery(session_tmp_path):
         assert (output_folder / "predictions.json").exists()
 
         keywords = {
+            r"^Writing run metadata at .*$": 1,
             r"^Loading dataset: * without caching. Trade-off: CPU RAM usage: less | Disk I/O: more$": 3,
             r"^Loading checkpoint from .*$": 1,
             r"^Restoring normalizer from checkpoint.$": 1,
@@ -317,7 +319,7 @@ def test_evaluate_pasa_montgomery(session_tmp_path):
         assert (output_folder / "evaluation.pdf").exists()
 
         keywords = {
-            r"^Saving evaluation metadata at .*$": 1,
+            r"^Writing run metadata at .*$": 1,
             r"^Setting --threshold=.*$": 1,
             r"^Computing performance on split .*...$": 3,
             r"^Saving evaluation results at .*$": 1,
@@ -335,6 +337,101 @@ def test_evaluate_pasa_montgomery(session_tmp_path):
             )
 
 
+@pytest.mark.slow
+@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
+def test_saliency_generation_pasa_montgomery(session_tmp_path):
+    from mednet.libs.classification.scripts.saliency.generate import generate
+    from mednet.libs.common.utils.checkpointer import (
+        CHECKPOINT_EXTENSION,
+        _get_checkpoint_from_alias,
+    )
+
+    runner = CliRunner()
+
+    with stdout_logging() as buf:
+        output_folder = session_tmp_path / "classification-standalone"
+        last = _get_checkpoint_from_alias(output_folder, "periodic")
+        assert last.name.endswith("epoch=0" + CHECKPOINT_EXTENSION)
+        result = runner.invoke(
+            generate,
+            [
+                "-vv",
+                "pasa",
+                "montgomery",
+                f"--weight={str(last)}",
+                f"--output-folder={str(output_folder)}",
+            ],
+        )
+        _assert_exit_0(result)
+
+        assert (output_folder / "saliency-generation.meta.json").exists()
+
+        keywords = {
+            r"^Writing run metadata at .*$": 1,
+            r"^Loading dataset:.*$": 3,
+            r"^Generating saliency maps for dataset .*$": 3,
+        }
+        buf.seek(0)
+        logging_output = buf.read()
+
+        for k, v in keywords.items():
+            assert _str_counter(k, logging_output) == v, (
+                f"Count for string '{k}' appeared "
+                f"({_str_counter(k, logging_output)}) "
+                f"instead of the expected {v}:\nOutput:\n{logging_output}"
+            )
+
+        assert len(list(output_folder.rglob("**/*.npy"))) == 138
+
+
+@pytest.mark.slow
+@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
+def test_saliency_view_pasa_montgomery(session_tmp_path):
+    from mednet.libs.classification.scripts.saliency.view import view
+    from mednet.libs.common.utils.checkpointer import (
+        CHECKPOINT_EXTENSION,
+        _get_checkpoint_from_alias,
+    )
+
+    runner = CliRunner()
+
+    with stdout_logging() as buf:
+        output_folder = session_tmp_path / "classification-standalone"
+        last = _get_checkpoint_from_alias(output_folder, "periodic")
+        assert last.name.endswith("epoch=0" + CHECKPOINT_EXTENSION)
+        result = runner.invoke(
+            view,
+            [
+                "-vv",
+                "pasa",
+                "montgomery",
+                f"--input-folder={str(output_folder)}",
+                f"--output-folder={str(output_folder)}",
+            ],
+        )
+        _assert_exit_0(result)
+
+        assert (output_folder / "saliency-view.meta.json").exists()
+
+        keywords = {
+            r"^Writing run metadata at .*$": 1,
+            r"^Loading dataset:.*$": 3,
+            r"^Generating visualizations for samples \(target = 1\) at dataset .*$": 3,
+        }
+        buf.seek(0)
+        logging_output = buf.read()
+
+        for k, v in keywords.items():
+            assert _str_counter(k, logging_output) == v, (
+                f"Count for string '{k}' appeared "
+                f"({_str_counter(k, logging_output)}) "
+                f"instead of the expected {v}:\nOutput:\n{logging_output}"
+            )
+
+        # there are only 58 samples with target = 1
+        assert len(list(output_folder.rglob("**/*.png"))) == 58
+
+
 @pytest.mark.slow
 @pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
 def test_train_pasa_montgomery_from_checkpoint(tmp_path):