Revamp tests; Delete outdated; Make common conftest.py; Improve segmentation...

Revamp tests; Delete outdated; Make common conftest.py; Improve segmentation tests; Allow segmentation tests to run on CI

pyproject.toml

@@ -151,7 +151,7 @@ test = "pytest -sv tests/"
 test-slow = { cmd = "pytest -sv tests/", env = { PYTEST_ADDOPTS = "-m slow" } }
 test-fast = { cmd = "pytest -sv tests/", env = { PYTEST_ADDOPTS = "-m '(not slow)'" } }
 test-cov = "pytest -sv --cov-report 'html:html/coverage' tests/"
-test-ci = "cp tests/classification/data/mednet.toml $XDG_CONFIG_HOME && pytest -sv --cov-report 'html:html/coverage' --cov-report 'xml:coverage.xml' --junitxml 'junit-coverage.xml' --ignore '.profile' tests/"
+test-ci = "cp tests/data/mednet.toml $XDG_CONFIG_HOME && pytest -sv --cov-report 'html:html/coverage' --cov-report 'xml:coverage.xml' --junitxml 'junit-coverage.xml' --ignore '.profile' tests/"
 
 [tool.pixi.feature.build.dependencies]
 hatch = "*"

tests/classification/data/lfs/.gitattributes

-_test_densenetrs_checkpoint.pth filter=lfs diff=lfs merge=lfs -text
-_test_fpasa_checkpoint.pth filter=lfs diff=lfs merge=lfs -text
-_test_logreg_checkpoint.pth filter=lfs diff=lfs merge=lfs -text
-_test_signstotb_checkpoint.pth filter=lfs diff=lfs merge=lfs -text
-_testdb.zip filter=lfs diff=lfs merge=lfs -text
-pasa.pth filter=lfs diff=lfs merge=lfs -text

tests/classification/data/lfs/.gitignore

-*~

tests/classification/data/lfs/README.md

-# Assets for Testing biosignal/software/mednet>
-
-This package contains test unit assets used by the test suit of
-biosignal/software/mednet>.
-
-
-## Updating
-
-To update the contents of this package, use [git-lfs](https://git-lfs.com),
-following the workflow described at the [GitLab support
-page](https://docs.gitlab.com/ee/topics/git/lfs/).
-
-Cloning and updating the repository, in particular, works the same as before
-and should not impose any workflow changes.

tests/classification/test_hivtb.py

@@ -103,6 +103,7 @@ def test_loading(database_checkers, name: str, dataset: str):
             prefixes=("HIV-TB_Algorithm_study_X-rays",),
             possible_labels=(0, 1),
             expected_num_labels=1,
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/classification/test_indian.py

@@ -108,6 +108,7 @@ def test_loading(database_checkers, name: str, dataset: str):
             prefixes=("DatasetA/Training", "DatasetA/Testing"),
             possible_labels=(0, 1),
             expected_num_labels=1,
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/classification/test_montgomery.py

@@ -108,6 +108,7 @@ def test_loading(database_checkers, name: str, dataset: str):
             prefixes=("CXR_png/MCUCXR_0",),
             possible_labels=(0, 1),
             expected_num_labels=1,
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/classification/test_nih_cxr14.py

@@ -83,6 +83,7 @@ def test_loading(database_checkers, name: str, dataset: str, num_labels: int):
             possible_labels=(0, 1),
             expected_num_labels=num_labels,
             expected_image_shape=(1, 1024, 1024),
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/classification/test_padchest.py

@@ -88,6 +88,7 @@ def test_loading(database_checkers, name: str, dataset: str, num_labels: int):
                 prefixes=("",),
                 possible_labels=(0, 1),
                 expected_num_labels=num_labels,
+                expected_meta_size=2,
             )
             limit -= 1
 

tests/classification/test_shenzhen.py

@@ -105,6 +105,7 @@ def test_loading(database_checkers, name: str, dataset: str):
             prefixes=("CXR_png/CHNCXR_0",),
             possible_labels=(0, 1),
             expected_num_labels=1,
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/classification/test_tbpoc.py

@@ -109,6 +109,7 @@ def test_loading(database_checkers, name: str, dataset: str):
             ),
             possible_labels=(0, 1),
             expected_num_labels=1,
+            expected_meta_size=2,
         )
         limit -= 1
 

tests/common/conftest.py

-# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import pathlib
-
-import pytest
-
-
-@pytest.fixture
-def datadir(request) -> pathlib.Path:
-    """Return the directory in which the test is sitting. Check the pytest
-    documentation for more information.
-
-    Parameters
-    ----------
-    request
-        Information of the requesting test function.
-
-    Returns
-    -------
-    pathlib.Path
-        The directory in which the test is sitting.
-    """
-
-    return pathlib.Path(request.module.__file__).parents[0] / "data"

tests/common/data/iris-test.csv

-5,3,1.6,0.2,Iris-setosa
-5,3.4,1.6,0.4,Iris-setosa
-5.2,3.5,1.5,0.2,Iris-setosa
-5.2,3.4,1.4,0.2,Iris-setosa
-4.7,3.2,1.6,0.2,Iris-setosa
-4.8,3.1,1.6,0.2,Iris-setosa
-5.4,3.4,1.5,0.4,Iris-setosa
-5.2,4.1,1.5,0.1,Iris-setosa
-5.5,4.2,1.4,0.2,Iris-setosa
-4.9,3.1,1.5,0.1,Iris-setosa
-5,3.2,1.2,0.2,Iris-setosa
-5.5,3.5,1.3,0.2,Iris-setosa
-4.9,3.1,1.5,0.1,Iris-setosa
-4.4,3,1.3,0.2,Iris-setosa
-5.1,3.4,1.5,0.2,Iris-setosa
-5,3.5,1.3,0.3,Iris-setosa
-4.5,2.3,1.3,0.3,Iris-setosa
-4.4,3.2,1.3,0.2,Iris-setosa
-5,3.5,1.6,0.6,Iris-setosa
-5.1,3.8,1.9,0.4,Iris-setosa
-4.8,3,1.4,0.3,Iris-setosa
-5.1,3.8,1.6,0.2,Iris-setosa
-4.6,3.2,1.4,0.2,Iris-setosa
-5.3,3.7,1.5,0.2,Iris-setosa
-5,3.3,1.4,0.2,Iris-setosa
-6.6,3,4.4,1.4,Iris-versicolor
-6.8,2.8,4.8,1.4,Iris-versicolor
-6.7,3,5,1.7,Iris-versicolor
-6,2.9,4.5,1.5,Iris-versicolor
-5.7,2.6,3.5,1,Iris-versicolor
-5.5,2.4,3.8,1.1,Iris-versicolor
-5.5,2.4,3.7,1,Iris-versicolor
-5.8,2.7,3.9,1.2,Iris-versicolor
-6,2.7,5.1,1.6,Iris-versicolor
-5.4,3,4.5,1.5,Iris-versicolor
-6,3.4,4.5,1.6,Iris-versicolor
-6.7,3.1,4.7,1.5,Iris-versicolor
-6.3,2.3,4.4,1.3,Iris-versicolor
-5.6,3,4.1,1.3,Iris-versicolor
-5.5,2.5,4,1.3,Iris-versicolor
-5.5,2.6,4.4,1.2,Iris-versicolor
-6.1,3,4.6,1.4,Iris-versicolor
-5.8,2.6,4,1.2,Iris-versicolor
-5,2.3,3.3,1,Iris-versicolor
-5.6,2.7,4.2,1.3,Iris-versicolor
-5.7,3,4.2,1.2,Iris-versicolor
-5.7,2.9,4.2,1.3,Iris-versicolor
-6.2,2.9,4.3,1.3,Iris-versicolor
-5.1,2.5,3,1.1,Iris-versicolor
-5.7,2.8,4.1,1.3,Iris-versicolor
-7.2,3.2,6,1.8,Iris-virginica
-6.2,2.8,4.8,1.8,Iris-virginica
-6.1,3,4.9,1.8,Iris-virginica
-6.4,2.8,5.6,2.1,Iris-virginica
-7.2,3,5.8,1.6,Iris-virginica
-7.4,2.8,6.1,1.9,Iris-virginica
-7.9,3.8,6.4,2,Iris-virginica
-6.4,2.8,5.6,2.2,Iris-virginica
-6.3,2.8,5.1,1.5,Iris-virginica
-6.1,2.6,5.6,1.4,Iris-virginica
-7.7,3,6.1,2.3,Iris-virginica
-6.3,3.4,5.6,2.4,Iris-virginica
-6.4,3.1,5.5,1.8,Iris-virginica
-6,3,4.8,1.8,Iris-virginica
-6.9,3.1,5.4,2.1,Iris-virginica
-6.7,3.1,5.6,2.4,Iris-virginica
-6.9,3.1,5.1,2.3,Iris-virginica
-5.8,2.7,5.1,1.9,Iris-virginica
-6.8,3.2,5.9,2.3,Iris-virginica
-6.7,3.3,5.7,2.5,Iris-virginica
-6.7,3,5.2,2.3,Iris-virginica
-6.3,2.5,5,1.9,Iris-virginica
-6.5,3,5.2,2,Iris-virginica
-6.2,3.4,5.4,2.3,Iris-virginica
-5.9,3,5.1,1.8,Iris-virginica

tests/common/data/iris-train.csv

-5.1,3.5,1.4,0.2,Iris-setosa
-4.9,3,1.4,0.2,Iris-setosa
-4.7,3.2,1.3,0.2,Iris-setosa
-4.6,3.1,1.5,0.2,Iris-setosa
-5,3.6,1.4,0.2,Iris-setosa
-5.4,3.9,1.7,0.4,Iris-setosa
-4.6,3.4,1.4,0.3,Iris-setosa
-5,3.4,1.5,0.2,Iris-setosa
-4.4,2.9,1.4,0.2,Iris-setosa
-4.9,3.1,1.5,0.1,Iris-setosa
-5.4,3.7,1.5,0.2,Iris-setosa
-4.8,3.4,1.6,0.2,Iris-setosa
-4.8,3,1.4,0.1,Iris-setosa
-4.3,3,1.1,0.1,Iris-setosa
-5.8,4,1.2,0.2,Iris-setosa
-5.7,4.4,1.5,0.4,Iris-setosa
-5.4,3.9,1.3,0.4,Iris-setosa
-5.1,3.5,1.4,0.3,Iris-setosa
-5.7,3.8,1.7,0.3,Iris-setosa
-5.1,3.8,1.5,0.3,Iris-setosa
-5.4,3.4,1.7,0.2,Iris-setosa
-5.1,3.7,1.5,0.4,Iris-setosa
-4.6,3.6,1,0.2,Iris-setosa
-5.1,3.3,1.7,0.5,Iris-setosa
-4.8,3.4,1.9,0.2,Iris-setosa
-7,3.2,4.7,1.4,Iris-versicolor
-6.4,3.2,4.5,1.5,Iris-versicolor
-6.9,3.1,4.9,1.5,Iris-versicolor
-5.5,2.3,4,1.3,Iris-versicolor
-6.5,2.8,4.6,1.5,Iris-versicolor
-5.7,2.8,4.5,1.3,Iris-versicolor
-6.3,3.3,4.7,1.6,Iris-versicolor
-4.9,2.4,3.3,1,Iris-versicolor
-6.6,2.9,4.6,1.3,Iris-versicolor
-5.2,2.7,3.9,1.4,Iris-versicolor
-5,2,3.5,1,Iris-versicolor
-5.9,3,4.2,1.5,Iris-versicolor
-6,2.2,4,1,Iris-versicolor
-6.1,2.9,4.7,1.4,Iris-versicolor
-5.6,2.9,3.6,1.3,Iris-versicolor
-6.7,3.1,4.4,1.4,Iris-versicolor
-5.6,3,4.5,1.5,Iris-versicolor
-5.8,2.7,4.1,1,Iris-versicolor
-6.2,2.2,4.5,1.5,Iris-versicolor
-5.6,2.5,3.9,1.1,Iris-versicolor
-5.9,3.2,4.8,1.8,Iris-versicolor
-6.1,2.8,4,1.3,Iris-versicolor
-6.3,2.5,4.9,1.5,Iris-versicolor
-6.1,2.8,4.7,1.2,Iris-versicolor
-6.4,2.9,4.3,1.3,Iris-versicolor
-6.3,3.3,6,2.5,Iris-virginica
-5.8,2.7,5.1,1.9,Iris-virginica
-7.1,3,5.9,2.1,Iris-virginica
-6.3,2.9,5.6,1.8,Iris-virginica
-6.5,3,5.8,2.2,Iris-virginica
-7.6,3,6.6,2.1,Iris-virginica
-4.9,2.5,4.5,1.7,Iris-virginica
-7.3,2.9,6.3,1.8,Iris-virginica
-6.7,2.5,5.8,1.8,Iris-virginica
-7.2,3.6,6.1,2.5,Iris-virginica
-6.5,3.2,5.1,2,Iris-virginica
-6.4,2.7,5.3,1.9,Iris-virginica
-6.8,3,5.5,2.1,Iris-virginica
-5.7,2.5,5,2,Iris-virginica
-5.8,2.8,5.1,2.4,Iris-virginica
-6.4,3.2,5.3,2.3,Iris-virginica
-6.5,3,5.5,1.8,Iris-virginica
-7.7,3.8,6.7,2.2,Iris-virginica
-7.7,2.6,6.9,2.3,Iris-virginica
-6,2.2,5,1.5,Iris-virginica
-6.9,3.2,5.7,2.3,Iris-virginica
-5.6,2.8,4.9,2,Iris-virginica
-7.7,2.8,6.7,2,Iris-virginica
-6.3,2.7,4.9,1.8,Iris-virginica
-6.7,3.3,5.7,2.1,Iris-virginica

tests/classification/conftest.py → tests/conftest.py

@@ -44,7 +44,7 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers",
         "skip_if_rc_var_not_set(name): this mark skips the test if a certain "
-        "~/.config/mednet.libs.classification.toml variable is not set",
+        "~/.config/mednet.toml variable is not set",
     )
 
     config.addinivalue_line("markers", "slow: this mark indicates slow tests")
@@ -134,14 +134,19 @@ class DatabaseCheckers:
 
             assert len(split[k]) == lengths[k]
             for s in split[k]:
-                assert any([s[0].startswith(k) for k in prefixes]), (
-                    f"Sample with name {s[0]} does not start with any of the "
-                    f"prefixes in {prefixes}"
-                )
-                if isinstance(s[1], list):
-                    assert all([k in possible_labels for k in s[1]])
-                else:
-                    assert s[1] in possible_labels
+                # check filename prefixes match, if prefixes were passed
+                if prefixes:
+                    assert any([s[0].startswith(k) for k in prefixes]), (
+                        f"Sample with name {s[0]} does not start with any of the "
+                        f"prefixes in {prefixes}"
+                    )
+
+                # check if labels match, if labels were passed
+                if possible_labels:
+                    if isinstance(s[1], list):
+                        assert all([k in possible_labels for k in s[1]])
+                    else:
+                        assert s[1] in possible_labels
 
     @staticmethod
     def check_loaded_batch(
@@ -151,6 +156,7 @@ class DatabaseCheckers:
         prefixes: typing.Sequence[str],
         possible_labels: typing.Sequence[int],
         expected_num_labels: int,
+        expected_meta_size: int,
         expected_image_shape: tuple[int, ...] | None = None,
     ):
         """Check the consistency of an individual (loaded) batch.
@@ -170,6 +176,9 @@ class DatabaseCheckers:
             These are the list of possible labels contained in any split.
         expected_num_labels
             The expected number of labels each sample should have.
+        expected_meta_size
+            The expected number of elements on the meta-information dictionary
+            on every sample.
         expected_image_shape
             The expected shape of the image (num_channels, width, height).
         """
@@ -186,18 +195,20 @@ class DatabaseCheckers:
             )
 
         assert isinstance(batch[1], dict)  # metadata
-        assert len(batch[1]) == 2  # label and name
+        assert len(batch[1]) == expected_meta_size
 
         assert "target" in batch[1]
-        assert all([k in possible_labels for k in batch[1]["target"]])
+        if possible_labels:
+            assert all([k in possible_labels for k in batch[1]["target"]])
 
         if expected_num_labels:
             assert len(batch[1]["target"]) == expected_num_labels
 
         assert "name" in batch[1]
-        assert all(
-            [any([k.startswith(j) for j in prefixes]) for k in batch[1]["name"]],
-        )
+        if prefixes:
+            assert all(
+                [any([k.startswith(j) for j in prefixes]) for k in batch[1]["name"]],
+            )
 
         # use the code below to view generated images
         # from torchvision.transforms.functional import to_pil_image