[data] Streamline test to make it easy to assess thresholds and visualize test samples

bob/ip/binseg/data/chasedb1/test.py

@@ -47,9 +47,8 @@ def test_loading():
 
     from ..utils import count_bw
     image_size = (999, 960)
-    bw_threshold_label = 0.10  #(vessels to background proportion limit)
 
-    def _check_sample(s):
+    def _check_sample(s, bw_threshold_label):
 
         data = s.data
         assert isinstance(data, dict)
@@ -79,14 +78,21 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
+        return w/b
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("default")
-    for s in subset["train"]: _check_sample(s)
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.08) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.10) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("second-annotation")
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.09) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.09) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
 
 @rc_variable_set('bob.ip.binseg.chasedb1.datadir')

bob/ip/binseg/data/drishtigs1/test.py

@@ -82,32 +82,33 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
-    subset = dataset.subsets("optic-cup-all")
+        return w/b
+
     limit = None
-    for s in subset["train"][:limit]:
-        _check_sample(s, 0.03)
-    for s in subset["test"][:limit]:
-        _check_sample(s, 0.04)
+    subset = dataset.subsets("optic-cup-all")
+    proportions = [_check_sample(s, 0.027) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.035) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("optic-disc-all")
-    for s in subset["train"][:limit]:
-        _check_sample(s, 0.05)
-    for s in subset["test"][:limit]:
-        _check_sample(s, 0.06)
+    proportions = [_check_sample(s, 0.045) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.055) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("optic-cup-any")
-    for s in subset["train"][:limit]:
-        _check_sample(s, 0.04)
-    for s in subset["test"][:limit]:
-        _check_sample(s, 0.05)
+    proportions = [_check_sample(s, 0.034) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.047) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("optic-disc-any")
-    for s in subset["train"][:limit]:
-        _check_sample(s, 0.055)
-    for s in subset["test"][:limit]:
-        _check_sample(s, 0.06)
+    proportions = [_check_sample(s, 0.052) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.060) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
 
 @rc_variable_set("bob.ip.binseg.drishtigs1.datadir")

bob/ip/binseg/data/drive/test.py

@@ -42,9 +42,8 @@ def test_loading():
 
     from ..utils import count_bw
     image_size = (565, 584)
-    bw_threshold_label = 0.14  #(vessels to background proportion limit)
 
-    def _check_sample(s):
+    def _check_sample(s, bw_threshold_label, bw_threshold_mask):
 
         data = s.data
         assert isinstance(data, dict)
@@ -62,7 +61,7 @@ def test_loading():
                 f"Counts of black + white ({b}+{w}) do not add up to total " \
                 f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
         assert (w/b) < bw_threshold_label, \
-                f"The proportion between black and white pixels " \
+                f"The proportion between black and white pixels in labels " \
                 f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
                 f"of {bw_threshold_label} at '{s.key}':label - this could " \
                 f"indicate a loading problem!"
@@ -70,13 +69,14 @@ def test_loading():
         assert "mask" in data
         nose.tools.eq_(data["mask"].size, image_size)
         nose.tools.eq_(data["mask"].mode, "1")
-        b, w = count_bw(data["mask"])
-        assert (b+w) == numpy.prod(image_size), \
-                f"Counts of black + white ({b}+{w}) do not add up to total " \
+        bm, wm = count_bw(data["mask"])
+        assert (bm+wm) == numpy.prod(image_size), \
+                f"Counts of black + white ({bm}+{wm}) do not add up to total " \
                 f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
-        assert w > b, \
-                f"The proportion between white and black pixels " \
-                f"({w} > {b}?) is not respected at '{s.key}':mask - " \
+        assert (wm/bm) > bw_threshold_mask, \
+                f"The proportion between black and white pixels in masks " \
+                f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+                f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
                 f"this could indicate a loading problem!"
 
         # to visualize images, uncomment the folowing code
@@ -87,14 +87,22 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"], data["mask"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
+        return w/b, wm/bm
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("default")
-    for s in subset["train"]: _check_sample(s)
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.14, 2.14) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+    proportions = [_check_sample(s, 0.12, 2.12) for s in subset["test"]][:limit]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
 
     subset = dataset.subsets("second-annotation")
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.12, 2.12) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
 
 
 @rc_variable_set('bob.ip.binseg.drive.datadir')

bob/ip/binseg/data/hrf/test.py

@@ -34,9 +34,8 @@ def test_loading():
 
     from ..utils import count_bw
     image_size = (3504, 2336)
-    bw_threshold_label = 0.12  #(vessels to background proportion limit)
 
-    def _check_sample(s):
+    def _check_sample(s, bw_threshold_label, bw_threshold_mask):
 
         data = s.data
         assert isinstance(data, dict)
@@ -62,13 +61,14 @@ def test_loading():
         assert "mask" in data
         nose.tools.eq_(data["mask"].size, image_size)
         nose.tools.eq_(data["mask"].mode, "1")
-        b, w = count_bw(data["mask"])
-        assert (b+w) == numpy.prod(image_size), \
-                f"Counts of black + white ({b}+{w}) do not add up to total " \
+        bm, wm = count_bw(data["mask"])
+        assert (bm+wm) == numpy.prod(image_size), \
+                f"Counts of black + white ({bm}+{wm}) do not add up to total " \
                 f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
-        assert w > b, \
-                f"The proportion between white and black pixels " \
-                f"({w} > {b}?) is not respected at '{s.key}':mask - " \
+        assert (wm/bm) > bw_threshold_mask, \
+                f"The proportion between black and white pixels in masks " \
+                f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+                f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
                 f"this could indicate a loading problem!"
 
         # to visualize images, uncomment the folowing code
@@ -79,11 +79,17 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"], data["mask"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
+        return w/b, wm/bm
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("default")
-    for s in subset["train"]: _check_sample(s)
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.12, 5.42) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+    proportions = [_check_sample(s, 0.12, 5.41) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
 
 
 @rc_variable_set('bob.ip.binseg.hrf.datadir')

bob/ip/binseg/data/iostar/test.py

@@ -48,7 +48,7 @@ def test_loading():
     from ..utils import count_bw
     image_size = (1024, 1024)
 
-    def _check_sample(s, bw_threshold_label):
+    def _check_sample(s, bw_threshold_label, bw_threshold_mask):
 
         data = s.data
         assert isinstance(data, dict)
@@ -74,13 +74,14 @@ def test_loading():
         assert "mask" in data
         nose.tools.eq_(data["mask"].size, image_size)
         nose.tools.eq_(data["mask"].mode, "1")
-        b, w = count_bw(data["mask"])
-        assert (b+w) == numpy.prod(image_size), \
-                f"Counts of black + white ({b}+{w}) do not add up to total " \
+        bm, wm = count_bw(data["mask"])
+        assert (bm+wm) == numpy.prod(image_size), \
+                f"Counts of black + white ({bm}+{wm}) do not add up to total " \
                 f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
-        assert w > b, \
-                f"The proportion between white and black pixels " \
-                f"({w} > {b}?) is not respected at '{s.key}':mask - " \
+        assert (wm/bm) > bw_threshold_mask, \
+                f"The proportion between black and white pixels in masks " \
+                f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+                f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
                 f"this could indicate a loading problem!"
 
         # to visualize images, uncomment the folowing code
@@ -91,17 +92,25 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"], data["mask"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
+        return w/b, wm/bm
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("vessel")
-    bw_threshold_label = 0.11  #(vessels to background proportion limit)
-    for s in subset["train"]: _check_sample(s, bw_threshold_label)
-    for s in subset["test"]: _check_sample(s, bw_threshold_label)
+    proportions = [_check_sample(s, 0.11, 3.19) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+    proportions = [_check_sample(s, 0.10, 3.27) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
 
     subset = dataset.subsets("optic-disc")
-    bw_threshold_label = 0.04  #(optic-disc to background proportion limit)
-    for s in subset["train"]: _check_sample(s, bw_threshold_label)
-    for s in subset["test"]: _check_sample(s, bw_threshold_label)
+    proportions = [_check_sample(s, 0.023, 3.19) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+    proportions = [_check_sample(s, 0.033, 3.27) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(k[0] for k in proportions)}")
+    #print(f"min mask proportions = {min(k[1] for k in proportions)}")
 
 @rc_variable_set('bob.ip.binseg.iostar.datadir')
 def test_check():

bob/ip/binseg/data/refuge/test.py

@@ -81,23 +81,25 @@ def test_loading():
         # display = overlayed_image(data["data"], data["label"])
         # display.show()
         # import ipdb; ipdb.set_trace()
-        # pass
 
+        return w/b
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("optic-disc")
-    for s in subset["train"]:
-        _check_sample(s, (2124, 2056), True, 3, 0.03)
-    for s in subset["validation"]:
-        _check_sample(s, (1634, 1634), False, 2, 0.045)
-    for s in subset["test"]:
-        _check_sample(s, (1634, 1634), True, 3, 0.03)
+    proportions = [_check_sample(s, (2124, 2056), True, 3, 0.029) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, (1634, 1634), False, 2, 0.043) for s in subset["validation"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, (1634, 1634), True, 3, 0.026) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("optic-cup")
-    for s in subset["train"]:
-        _check_sample(s, (2124, 2056), True, 3, 0.018)
-    for s in subset["validation"]:
-        _check_sample(s, (1634, 1634), False, 2, 0.035)
-    for s in subset["test"]:
-        _check_sample(s, (1634, 1634), True, 3, 0.018)
+    proportions = [_check_sample(s, (2124, 2056), True, 3, 0.018) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, (1634, 1634), False, 2, 0.030) for s in subset["validation"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, (1634, 1634), True, 3, 0.017) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
 
 @rc_variable_set("bob.ip.binseg.refuge.datadir")

bob/ip/binseg/data/stare/test.py

@@ -47,9 +47,8 @@ def test_loading():
 
     from ..utils import count_bw
     image_size = (700, 605)
-    bw_threshold_label = 0.19  #(vessels to background proportion limit)
 
-    def _check_sample(s):
+    def _check_sample(s, bw_threshold_label):
 
         data = s.data
         assert isinstance(data, dict)
@@ -79,14 +78,21 @@ def test_loading():
         #display = overlayed_image(data["data"], data["label"])
         #display.show()
         #import ipdb; ipdb.set_trace()
-        #pass
 
+        return w/b
+
+    limit = None  #use this to limit testing to first images only
     subset = dataset.subsets("default")
-    for s in subset["train"]: _check_sample(s)
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.10) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.12) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
     subset = dataset.subsets("second-annotation")
-    for s in subset["test"]: _check_sample(s)
+    proportions = [_check_sample(s, 0.19) for s in subset["train"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
+    proportions = [_check_sample(s, 0.18) for s in subset["test"][:limit]]
+    #print(f"max label proportions = {max(proportions)}")
 
 
 @rc_variable_set('bob.ip.binseg.stare.datadir')