[segmentation] Add drhagis database

.gitattributes

+src/mednet/libs/segmentation/config/data/cxr8/default.json filter=lfs diff=lfs merge=lfs -text

pyproject.toml

@@ -430,6 +430,9 @@ chasedb1-2nd = "mednet.libs.segmentation.config.data.chasedb1.second_annotator"
 # cxr8 - cxr
 cxr8 = "mednet.libs.segmentation.config.data.cxr8.default"
 
+# drhagis dataset - retinography
+drhagis = "mednet.libs.segmentation.config.data.drhagis.default"
+
 # drive dataset - retinography
 drive = "mednet.libs.segmentation.config.data.drive.default"
 

src/mednet/libs/segmentation/config/data/drhagis/datamodule.py

+# SPDX-FileCopyrightText: Copyright © 2024 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""DRHAGIS dataset for Vessel Segmentation."""
+
+import os
+import pathlib
+
+import PIL.Image
+from mednet.libs.common.data.datamodule import CachingDataModule
+from mednet.libs.common.data.split import make_split
+from mednet.libs.common.data.typing import Sample
+from mednet.libs.common.models.transforms import crop_image_to_mask
+from mednet.libs.segmentation.data.typing import (
+    SegmentationRawDataLoader as _SegmentationRawDataLoader,
+)
+from torchvision import tv_tensors
+from torchvision.transforms.functional import to_tensor
+
+from ....utils.rc import load_rc
+
+CONFIGURATION_KEY_DATADIR = "datadir." + (__name__.rsplit(".", 2)[-2])
+"""Key to search for in the configuration file for the root directory of this
+database."""
+
+
+class SegmentationRawDataLoader(_SegmentationRawDataLoader):
+    """A specialized raw-data-loader for the Drive dataset."""
+
+    datadir: pathlib.Path
+    """This variable contains the base directory where the database raw data is
+    stored."""
+
+    def __init__(self):
+        self.datadir = pathlib.Path(
+            load_rc().get(CONFIGURATION_KEY_DATADIR, os.path.realpath(os.curdir))
+        )
+
+    def sample(self, sample: tuple[str, str, str]) -> Sample:
+        """Load a single image sample from the disk.
+
+        Parameters
+        ----------
+        sample
+            A tuple containing path suffixes to the sample image, target, and mask
+            to be loaded, within the dataset root folder.
+
+        Returns
+        -------
+            The sample representation.
+        """
+
+        image = to_tensor(PIL.Image.open(self.datadir / sample[0]).convert(mode="RGB"))
+        target = to_tensor(
+            PIL.Image.open(self.datadir / sample[1]).convert(mode="1", dither=None)
+        )
+        mask = to_tensor(
+            PIL.Image.open(self.datadir / sample[2]).convert(mode="1", dither=None)
+        )
+
+        tensor = tv_tensors.Image(crop_image_to_mask(image, mask))
+        target = tv_tensors.Image(crop_image_to_mask(target, mask))
+        mask = tv_tensors.Mask(crop_image_to_mask(mask, mask))
+
+        return tensor, dict(target=target, mask=mask, name=sample[0])  # type: ignore[arg-type]
+
+
+class DataModule(CachingDataModule):
+    """DRHAGIS dataset for Vessel Segmentation.
+
+    The DR HAGIS database has been created to aid the development of vessel extraction algorithms
+    suitable for retinal screening programmes. Researchers are encouraged to test their
+    segmentation algorithms using this database.
+
+    It should be noted that image 24 and 32 are identical, as this fundus image was obtained
+    from a patient exhibiting both diabetic retinopathy and age-related macular degeneration.
+
+    The images resolutions (height x width) are:
+        - 4752x3168  or
+        - 3456x2304  or
+        - 3126x2136  or
+        - 2896x1944  or
+        - 2816x1880  or
+
+    * Protocol ``default``:
+
+    * Training samples: 19 (including labels and masks)
+    * Test samples: 20 (including labels and masks)
+
+    Parameters
+    ----------
+    split_filename
+        Name of the .json file containing the split to load.
+    """
+
+    def __init__(self, split_filename: str):
+        assert __package__ is not None
+        super().__init__(
+            database_split=make_split(__package__, split_filename),
+            raw_data_loader=SegmentationRawDataLoader(),
+            database_name=__package__.rsplit(".", 1)[1],
+            split_name=pathlib.Path(split_filename).stem,
+        )

src/mednet/libs/segmentation/config/data/drhagis/default.json

+{
+	"train": [
+		[
+			"Fundus_Images/1.jpg",
+			"Manual_Segmentations/1_manual_orig.png",
+			"Mask_images/1_mask_orig.png"
+		],
+		[
+			"Fundus_Images/2.jpg",
+			"Manual_Segmentations/2_manual_orig.png",
+			"Mask_images/2_mask_orig.png"
+		],
+		[
+			"Fundus_Images/3.jpg",
+			"Manual_Segmentations/3_manual_orig.png",
+			"Mask_images/3_mask_orig.png"
+		],
+		[
+			"Fundus_Images/4.jpg",
+			"Manual_Segmentations/4_manual_orig.png",
+			"Mask_images/4_mask_orig.png"
+		],
+		[
+			"Fundus_Images/5.jpg",
+			"Manual_Segmentations/5_manual_orig.png",
+			"Mask_images/5_mask_orig.png"
+		],
+		[
+			"Fundus_Images/11.jpg",
+			"Manual_Segmentations/11_manual_orig.png",
+			"Mask_images/11_mask_orig.png"
+		],
+		[
+			"Fundus_Images/12.jpg",
+			"Manual_Segmentations/12_manual_orig.png",
+			"Mask_images/12_mask_orig.png"
+		],
+		[
+			"Fundus_Images/13.jpg",
+			"Manual_Segmentations/13_manual_orig.png",
+			"Mask_images/13_mask_orig.png"
+		],
+		[
+			"Fundus_Images/14.jpg",
+			"Manual_Segmentations/14_manual_orig.png",
+			"Mask_images/14_mask_orig.png"
+		],
+		[
+			"Fundus_Images/15.jpg",
+			"Manual_Segmentations/15_manual_orig.png",
+			"Mask_images/15_mask_orig.png"
+		],
+		[
+			"Fundus_Images/21.jpg",
+			"Manual_Segmentations/21_manual_orig.png",
+			"Mask_images/21_mask_orig.png"
+		],
+		[
+			"Fundus_Images/22.jpg",
+			"Manual_Segmentations/22_manual_orig.png",
+			"Mask_images/22_mask_orig.png"
+		],
+		[
+			"Fundus_Images/23.jpg",
+			"Manual_Segmentations/23_manual_orig.png",
+			"Mask_images/23_mask_orig.png"
+		],
+		[
+			"Fundus_Images/24.jpg",
+			"Manual_Segmentations/24_manual_orig.png",
+			"Mask_images/24_mask_orig.png"
+		],
+		[
+			"Fundus_Images/25.jpg",
+			"Manual_Segmentations/25_manual_orig.png",
+			"Mask_images/25_mask_orig.png"
+		],
+		[
+			"Fundus_Images/31.jpg",
+			"Manual_Segmentations/31_manual_orig.png",
+			"Mask_images/31_mask_orig.png"
+		],
+		[
+			"Fundus_Images/33.jpg",
+			"Manual_Segmentations/33_manual_orig.png",
+			"Mask_images/33_mask_orig.png"
+		],
+		[
+			"Fundus_Images/34.jpg",
+			"Manual_Segmentations/34_manual_orig.png",
+			"Mask_images/34_mask_orig.png"
+		],
+		[
+			"Fundus_Images/35.jpg",
+			"Manual_Segmentations/35_manual_orig.png",
+			"Mask_images/35_mask_orig.png"
+		]
+	],
+	"test": [
+		[
+			"Fundus_Images/6.jpg",
+			"Manual_Segmentations/6_manual_orig.png",
+			"Mask_images/6_mask_orig.png"
+		],
+		[
+			"Fundus_Images/7.jpg",
+			"Manual_Segmentations/7_manual_orig.png",
+			"Mask_images/7_mask_orig.png"
+		],
+		[
+			"Fundus_Images/8.jpg",
+			"Manual_Segmentations/8_manual_orig.png",
+			"Mask_images/8_mask_orig.png"
+		],
+		[
+			"Fundus_Images/9.jpg",
+			"Manual_Segmentations/9_manual_orig.png",
+			"Mask_images/9_mask_orig.png"
+		],
+		[
+			"Fundus_Images/10.jpg",
+			"Manual_Segmentations/10_manual_orig.png",
+			"Mask_images/10_mask_orig.png"
+		],
+		[
+			"Fundus_Images/16.jpg",
+			"Manual_Segmentations/16_manual_orig.png",
+			"Mask_images/16_mask_orig.png"
+		],
+		[
+			"Fundus_Images/17.jpg",
+			"Manual_Segmentations/17_manual_orig.png",
+			"Mask_images/17_mask_orig.png"
+		],
+		[
+			"Fundus_Images/18.jpg",
+			"Manual_Segmentations/18_manual_orig.png",
+			"Mask_images/18_mask_orig.png"
+		],
+		[
+			"Fundus_Images/19.jpg",
+			"Manual_Segmentations/19_manual_orig.png",
+			"Mask_images/19_mask_orig.png"
+		],
+		[
+			"Fundus_Images/20.jpg",
+			"Manual_Segmentations/20_manual_orig.png",
+			"Mask_images/20_mask_orig.png"
+		],
+		[
+			"Fundus_Images/26.jpg",
+			"Manual_Segmentations/26_manual_orig.png",
+			"Mask_images/26_mask_orig.png"
+		],
+		[
+			"Fundus_Images/27.jpg",
+			"Manual_Segmentations/27_manual_orig.png",
+			"Mask_images/27_mask_orig.png"
+		],
+		[
+			"Fundus_Images/28.jpg",
+			"Manual_Segmentations/28_manual_orig.png",
+			"Mask_images/28_mask_orig.png"
+		],
+		[
+			"Fundus_Images/29.jpg",
+			"Manual_Segmentations/29_manual_orig.png",
+			"Mask_images/29_mask_orig.png"
+		],
+		[
+			"Fundus_Images/30.jpg",
+			"Manual_Segmentations/30_manual_orig.png",
+			"Mask_images/30_mask_orig.png"
+		],
+		[
+			"Fundus_Images/36.jpg",
+			"Manual_Segmentations/36_manual_orig.png",
+			"Mask_images/36_mask_orig.png"
+		],
+		[
+			"Fundus_Images/37.jpg",
+			"Manual_Segmentations/37_manual_orig.png",
+			"Mask_images/37_mask_orig.png"
+		],
+		[
+			"Fundus_Images/38.jpg",
+			"Manual_Segmentations/38_manual_orig.png",
+			"Mask_images/38_mask_orig.png"
+		],
+		[
+			"Fundus_Images/39.jpg",
+			"Manual_Segmentations/39_manual_orig.png",
+			"Mask_images/39_mask_orig.png"
+		],
+		[
+			"Fundus_Images/40.jpg",
+			"Manual_Segmentations/40_manual_orig.png",
+			"Mask_images/40_mask_orig.png"
+		]
+	]
+}

src/mednet/libs/segmentation/config/data/drhagis/default.py

+# SPDX-FileCopyrightText: Copyright © 2024 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""DRHAGIS dataset for Vessel Segmentation (default protocol).
+
+* This configuration resolution: 1760 x 1760 (Resizing)
+"""
+
+from mednet.libs.segmentation.config.data.drhagis.datamodule import DataModule
+
+datamodule = DataModule("default.json")

src/mednet/libs/segmentation/tests/test_drhagis.py

+# SPDX-FileCopyrightText: Copyright © 2024 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Tests for drhagis dataset."""
+
+import importlib
+
+import pytest
+from click.testing import CliRunner
+
+
+def id_function(val):
+    if isinstance(val, dict):
+        return str(val)
+    return repr(val)
+
+
+@pytest.mark.parametrize(
+    "split,lengths",
+    [
+        ("default", dict(train=19, test=20)),
+    ],
+    ids=id_function,  # just changes how pytest prints it
+)
+def test_protocol_consistency(
+    database_checkers,
+    split: str,
+    lengths: dict[str, int],
+):
+    from mednet.libs.common.data.split import make_split
+
+    database_checkers.check_split(
+        make_split("mednet.libs.segmentation.config.data.drhagis", f"{split}.json"),
+        lengths=lengths,
+    )
+
+
+@pytest.mark.skip_if_rc_var_not_set("datadir.drhagis")
+def test_database_check():
+    from mednet.libs.segmentation.scripts.database import check
+
+    runner = CliRunner()
+    result = runner.invoke(check, ["drhagis"])
+    assert (
+        result.exit_code == 0
+    ), f"Exit code {result.exit_code} != 0 -- Output:\n{result.output}"
+
+
+@pytest.mark.skip_if_rc_var_not_set("datadir.drhagis")
+@pytest.mark.parametrize(
+    "dataset",
+    [
+        "train",
+        "test",
+    ],
+)
+@pytest.mark.parametrize(
+    "name",
+    [
+        "default",
+    ],
+)
+def test_loading(database_checkers, name: str, dataset: str):
+    datamodule = importlib.import_module(
+        f".{name}",
+        "mednet.libs.segmentation.config.data.drhagis",
+    ).datamodule
+
+    datamodule.model_transforms = []  # should be done before setup()
+    datamodule.setup("predict")  # sets up all datasets
+
+    loader = datamodule.predict_dataloader()[dataset]
+
+    limit = 3  # limit load checking
+    for batch in loader:
+        if limit == 0:
+            break
+        database_checkers.check_loaded_batch(
+            batch,
+            batch_size=1,
+            color_planes=3,
+            expected_num_targets=1,
+        )
+        limit -= 1
+
+
+@pytest.mark.skip_if_rc_var_not_set("datadir.drhagis")
+def test_raw_transforms_image_quality(database_checkers, datadir):
+    reference_histogram_file = str(
+        datadir / "histograms/raw_data/histograms_drhagis_default.json",
+    )
+
+    datamodule = importlib.import_module(
+        ".default",
+        "mednet.libs.segmentation.config.data.drhagis",
+    ).datamodule
+
+    datamodule.model_transforms = []
+    datamodule.setup("predict")
+
+    database_checkers.check_image_quality(datamodule, reference_histogram_file)
+
+
+@pytest.mark.skip_if_rc_var_not_set("datadir.drhagis")
+@pytest.mark.parametrize(
+    "model_name",
+    ["lwnet"],
+)
+def test_model_transforms_image_quality(database_checkers, datadir, model_name):
+    reference_histogram_file = str(
+        datadir / f"histograms/models/histograms_{model_name}_drhagis_default.json",
+    )
+
+    datamodule = importlib.import_module(
+        ".default",
+        "mednet.libs.segmentation.config.data.drhagis",
+    ).datamodule
+
+    model = importlib.import_module(
+        f".{model_name}",
+        "mednet.libs.segmentation.config.models",
+    ).model
+
+    datamodule.model_transforms = model.model_transforms
+    datamodule.setup("predict")
+
+    database_checkers.check_image_quality(
+        datamodule,
+        reference_histogram_file,
+        compare_type="statistical",
+        pearson_coeff_threshold=0.005,
+    )