diff --git a/bob/ip/binseg/configs/datasets/JSRT/xtest.py b/bob/ip/binseg/configs/datasets/JSRT/xtest.py
deleted file mode 100644
index 81d2fe2aa6599c41b4140bb59578705f943ad565..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/JSRT/xtest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-"""JSRT cross-evaluation dataset
-"""
-
-from bob.ip.binseg.configs.datasets.JSRT.default import dataset as _jsrt
-from bob.ip.binseg.configs.datasets.MC.default import dataset as _mc
-from bob.ip.binseg.configs.datasets.Shenzhen.default import dataset as _shenzhen
-
-dataset = {
- "train": _jsrt["train"],
- "validation": _jsrt["validation"],
- "test": _jsrt["test"],
- "MC (train)": _mc["train"],
- "MC (validation)": _mc["validation"],
- "MC (test)": _mc["test"],
- "Shenzhen (train)": _shenzhen["train"],
- "Shenzhen (validation)": _shenzhen["validation"],
- "Shenzhen (test)": _shenzhen["test"],
-}
diff --git a/bob/ip/binseg/configs/datasets/MC/xtest.py b/bob/ip/binseg/configs/datasets/MC/xtest.py
deleted file mode 100644
index 0a308cd3ddf9787ced56ff60459b956ef80e7f07..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/MC/xtest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-"""MC cross-evaluation dataset
-"""
-
-from bob.ip.binseg.configs.datasets.JSRT.default import dataset as _jsrt
-from bob.ip.binseg.configs.datasets.MC.default import dataset as _mc
-from bob.ip.binseg.configs.datasets.Shenzhen.default import dataset as _shenzhen
-
-dataset = {
- "train": _mc["train"],
- "validation": _mc["validation"],
- "test": _mc["test"],
- "JSRT (train)": _jsrt["train"],
- "JSRT (validation)": _jsrt["validation"],
- "JSRT (test)": _jsrt["test"],
- "Shenzhen (train)": _shenzhen["train"],
- "Shenzhen (validation)": _shenzhen["validation"],
- "Shenzhen (test)": _shenzhen["test"],
-}
diff --git a/bob/ip/binseg/configs/datasets/Shenzhen/__init__.py b/bob/ip/binseg/configs/datasets/Shenzhen/__init__.py
deleted file mode 100644
index 481afa61ad34a5474beb828d1464aff63ca71416..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/Shenzhen/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-
-def _maker(protocol):
-
- from ....data.Shenzhen import dataset as raw
- from ....data.transforms import Resize
- from .. import make_dataset as mk
-
- return mk(raw.subsets(protocol), [Resize((512, 512))])
-
-
-def _maker_256(protocol):
-
- from ....data.Shenzhen import dataset as raw
- from ....data.transforms import Resize
- from .. import make_dataset as mk
-
- return mk(raw.subsets(protocol), [Resize((256, 256))])
diff --git a/bob/ip/binseg/configs/datasets/Shenzhen/default.py b/bob/ip/binseg/configs/datasets/Shenzhen/default.py
deleted file mode 100644
index 0a82ef34e9355e3fc864c830d5fa3d28997c5a69..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/Shenzhen/default.py
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-"""Shenzhen dataset for Lung Segmentation (default protocol)
-
-* Split reference: [GAÁL-2020]_
-* Configuration resolution: 512 x 512
-* See :py:mod:`bob.ip.binseg.data.Shenzhen` for dataset details
-"""
-
-from bob.ip.binseg.configs.datasets.Shenzhen import _maker
-
-dataset = _maker("default")
diff --git a/bob/ip/binseg/configs/datasets/Shenzhen/default_256.py b/bob/ip/binseg/configs/datasets/Shenzhen/default_256.py
deleted file mode 100644
index 468a00e69efbe2f8dd4396ba39dc34aaf28a6ed6..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/Shenzhen/default_256.py
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-"""Shenzhen dataset for Lung Segmentation (default protocol)
-
-* Split reference: [GAÁL-2020]_
-* Configuration resolution: 512 x 512
-* See :py:mod:`bob.ip.binseg.data.Shenzhen` for dataset details
-"""
-
-from bob.ip.binseg.configs.datasets.Shenzhen import _maker_256
-
-dataset = _maker_256("default")
diff --git a/bob/ip/binseg/configs/datasets/Shenzhen/xtest.py b/bob/ip/binseg/configs/datasets/Shenzhen/xtest.py
deleted file mode 100644
index 6a42c67de756b23927c38f7605b7db1e25f4e363..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/Shenzhen/xtest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-"""Shenzhen cross-evaluation dataset
-"""
-
-from bob.ip.binseg.configs.datasets.JSRT.default import dataset as _jsrt
-from bob.ip.binseg.configs.datasets.MC.default import dataset as _mc
-from bob.ip.binseg.configs.datasets.Shenzhen.default import dataset as _shenzhen
-
-dataset = {
- "train": _shenzhen["train"],
- "validation": _shenzhen["validation"],
- "test": _shenzhen["test"],
- "MC (train)": _mc["train"],
- "MC (validation)": _mc["validation"],
- "MC (test)": _mc["test"],
- "JSRT (train)": _jsrt["train"],
- "JSRT (validation)": _jsrt["validation"],
- "JSRT (test)": _jsrt["test"],
-}
diff --git a/bob/ip/binseg/configs/datasets/JSRT/__init__.py b/bob/ip/binseg/configs/datasets/jsrt/__init__.py
similarity index 81%
rename from bob/ip/binseg/configs/datasets/JSRT/__init__.py
rename to bob/ip/binseg/configs/datasets/jsrt/__init__.py
index 2d5ca8481c24b9b711a8a5403537a9fb97904bdc..35453e83e5d9616f66fce646c4c42d0f45abcb1f 100644
--- a/bob/ip/binseg/configs/datasets/JSRT/__init__.py
+++ b/bob/ip/binseg/configs/datasets/jsrt/__init__.py
@@ -4,7 +4,7 @@
def _maker(protocol):
- from ....data.JSRT import dataset as raw
+ from ....data.jsrt import dataset as raw
from ....data.transforms import Resize
from .. import make_dataset as mk
diff --git a/bob/ip/binseg/configs/datasets/JSRT/default.py b/bob/ip/binseg/configs/datasets/jsrt/default.py
similarity index 58%
rename from bob/ip/binseg/configs/datasets/JSRT/default.py
rename to bob/ip/binseg/configs/datasets/jsrt/default.py
index c8c6fabf4623f962a89bb446f68f5325e46749ce..0507092be956276fb2438f3bbbce12aef30172ec 100644
--- a/bob/ip/binseg/configs/datasets/JSRT/default.py
+++ b/bob/ip/binseg/configs/datasets/jsrt/default.py
@@ -3,11 +3,11 @@
"""Japanese Society of Radiological Technology dataset for Lung Segmentation (default protocol)
-* Split reference: [GAÁL-2020]_
+* Split reference: [GAAL-2020]_
* Configuration resolution: 512 x 512
-* See :py:mod:`bob.ip.binseg.data.JSRT` for dataset details
+* See :py:mod:`bob.ip.binseg.data.jsrt` for dataset details
"""
-from bob.ip.binseg.configs.datasets.JSRT import _maker
+from bob.ip.binseg.configs.datasets.jsrt import _maker
dataset = _maker("default")
diff --git a/bob/ip/binseg/configs/datasets/jsrt/xtest.py b/bob/ip/binseg/configs/datasets/jsrt/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..b14919ece180435d462fc9c3b6a91e4d3e6d6ac2
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/jsrt/xtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""JSRT CXR cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.jsrt.default import dataset as _jsrt
+from bob.ip.binseg.configs.datasets.montgomery.default import dataset as _mc
+from bob.ip.binseg.configs.datasets.shenzhen.default import dataset as _shenzhen
+
+dataset = {
+ "train": _jsrt["train"],
+ "validation": _jsrt["validation"],
+ "test": _jsrt["test"],
+ "montgomery (train)": _mc["train"],
+ "montgomery (validation)": _mc["validation"],
+ "montgomery (test)": _mc["test"],
+ "shenzhen (train)": _shenzhen["train"],
+ "shenzhen (validation)": _shenzhen["validation"],
+ "shenzhen (test)": _shenzhen["test"],
+}
diff --git a/bob/ip/binseg/configs/datasets/MC/__init__.py b/bob/ip/binseg/configs/datasets/montgomery/__init__.py
similarity index 79%
rename from bob/ip/binseg/configs/datasets/MC/__init__.py
rename to bob/ip/binseg/configs/datasets/montgomery/__init__.py
index b4aacadfd7960a5334dea75c1313a91fd4d2410d..54ae569accd8f7211fc6299bc4eff0c356ec2251 100644
--- a/bob/ip/binseg/configs/datasets/MC/__init__.py
+++ b/bob/ip/binseg/configs/datasets/montgomery/__init__.py
@@ -4,7 +4,7 @@
def _maker(protocol):
- from ....data.MC import dataset as raw
+ from ....data.montgomery import dataset as raw
from ....data.transforms import Resize
from .. import make_dataset as mk
diff --git a/bob/ip/binseg/configs/datasets/MC/default.py b/bob/ip/binseg/configs/datasets/montgomery/default.py
similarity index 53%
rename from bob/ip/binseg/configs/datasets/MC/default.py
rename to bob/ip/binseg/configs/datasets/montgomery/default.py
index 0d06272c01650c9074048a3f052cd491c4d1de85..96f941411b4e72b4822d670542c5b6dfc955142c 100644
--- a/bob/ip/binseg/configs/datasets/MC/default.py
+++ b/bob/ip/binseg/configs/datasets/montgomery/default.py
@@ -3,11 +3,11 @@
"""Montgomery County dataset for Lung Segmentation (default protocol)
-* Split reference: [GAÁL-2020]_
+* Split reference: [GAAL-2020]_
* Configuration resolution: 512 x 512
-* See :py:mod:`bob.ip.binseg.data.MC` for dataset details
+* See :py:mod:`bob.ip.binseg.data.montgomery` for dataset details
"""
-from bob.ip.binseg.configs.datasets.MC import _maker
+from bob.ip.binseg.configs.datasets.montgomery import _maker
dataset = _maker("default")
diff --git a/bob/ip/binseg/configs/datasets/montgomery/xtest.py b/bob/ip/binseg/configs/datasets/montgomery/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..c350b6035a45a2dde984047fd6c773277f09a2d6
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/montgomery/xtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Montgomery County cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.jsrt.default import dataset as _jsrt
+from bob.ip.binseg.configs.datasets.montgomery.default import dataset as _mc
+from bob.ip.binseg.configs.datasets.shenzhen.default import dataset as _shenzhen
+
+dataset = {
+ "train": _mc["train"],
+ "validation": _mc["validation"],
+ "test": _mc["test"],
+ "jsrt (train)": _jsrt["train"],
+ "jsrt (validation)": _jsrt["validation"],
+ "jsrt (test)": _jsrt["test"],
+ "shenzhen (train)": _shenzhen["train"],
+ "shenzhen (validation)": _shenzhen["validation"],
+ "shenzhen (test)": _shenzhen["test"],
+}
diff --git a/bob/ip/binseg/configs/datasets/shenzhen/__init__.py b/bob/ip/binseg/configs/datasets/shenzhen/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0887bb50cf6b3beb130fffaeafcc2bbd469b5af
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/shenzhen/__init__.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+def _maker(protocol, n):
+
+ from ....data.shenzhen import dataset as raw
+ from ....data.transforms import Resize
+ from .. import make_dataset as mk
+
+ return mk(raw.subsets(protocol), [Resize((n, n))])
diff --git a/bob/ip/binseg/configs/datasets/shenzhen/default.py b/bob/ip/binseg/configs/datasets/shenzhen/default.py
new file mode 100644
index 0000000000000000000000000000000000000000..b74107a0068dce4549b9355b3359a602bad85e47
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/shenzhen/default.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Shenzhen dataset for Lung Segmentation (default protocol)
+
+* Split reference: [GAAL-2020]_
+* Configuration resolution: 512 x 512
+* See :py:mod:`bob.ip.binseg.data.shenzhen` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.shenzhen import _maker
+
+dataset = _maker("default", 512)
diff --git a/bob/ip/binseg/configs/datasets/shenzhen/default_256.py b/bob/ip/binseg/configs/datasets/shenzhen/default_256.py
new file mode 100644
index 0000000000000000000000000000000000000000..2152e17896bcc8d8939d8d0262c544833dda8238
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/shenzhen/default_256.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Shenzhen dataset for Lung Segmentation (default protocol)
+
+* Split reference: [GAAL-2020]_
+* Configuration resolution: 256 x 256 pixels
+* See :py:mod:`bob.ip.binseg.data.shenzhen` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.shenzhen import _maker
+
+dataset = _maker("default", 256)
diff --git a/bob/ip/binseg/configs/datasets/shenzhen/xtest.py b/bob/ip/binseg/configs/datasets/shenzhen/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..52da00da0dfeb4895271070618a2e4cb28e283b2
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/shenzhen/xtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Shenzhen cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.jsrt.default import dataset as _jsrt
+from bob.ip.binseg.configs.datasets.montgomery.default import dataset as _mc
+from bob.ip.binseg.configs.datasets.shenzhen.default import dataset as _shenzhen
+
+dataset = {
+ "train": _shenzhen["train"],
+ "validation": _shenzhen["validation"],
+ "test": _shenzhen["test"],
+ "montgomery (train)": _mc["train"],
+ "montgomery (validation)": _mc["validation"],
+ "montgomery (test)": _mc["test"],
+ "jsrt (train)": _jsrt["train"],
+ "jsrt (validation)": _jsrt["validation"],
+ "jsrt (test)": _jsrt["test"],
+}
diff --git a/bob/ip/binseg/data/JSRT/__init__.py b/bob/ip/binseg/data/jsrt/__init__.py
similarity index 94%
rename from bob/ip/binseg/data/JSRT/__init__.py
rename to bob/ip/binseg/data/jsrt/__init__.py
index 42a26cc4070294415dac0fb6551e229da20e8403..bd913a513f9613f46dd77849bcb537830fbe79d0 100644
--- a/bob/ip/binseg/data/JSRT/__init__.py
+++ b/bob/ip/binseg/data/jsrt/__init__.py
@@ -10,7 +10,7 @@ available.
* Reference: [JSRT-2000]_
* Original resolution (height x width): 2048 x 2048
* Configuration resolution: 512 x 512 (after rescaling)
-* Split reference: [GAÁL-2020]_
+* Split reference: [GAAL-2020]_
* Protocol ``default``:
* Training samples: 172 (including labels)
@@ -36,7 +36,7 @@ _protocols = [
]
_root_path = bob.extension.rc.get(
- "bob.ip.binseg.JSRT.datadir", os.path.realpath(os.curdir)
+ "bob.ip.binseg.jsrt.datadir", os.path.realpath(os.curdir)
)
diff --git a/bob/ip/binseg/data/JSRT/default.json b/bob/ip/binseg/data/jsrt/default.json
similarity index 100%
rename from bob/ip/binseg/data/JSRT/default.json
rename to bob/ip/binseg/data/jsrt/default.json
diff --git a/bob/ip/binseg/data/MC/__init__.py b/bob/ip/binseg/data/montgomery/__init__.py
similarity index 94%
rename from bob/ip/binseg/data/MC/__init__.py
rename to bob/ip/binseg/data/montgomery/__init__.py
index d479eb6158ab704c9e4303e6966d8ee3be526010..b1d7844bee70e1ed21e5389624650df9584d4bce 100644
--- a/bob/ip/binseg/data/MC/__init__.py
+++ b/bob/ip/binseg/data/montgomery/__init__.py
@@ -10,7 +10,7 @@ normal cases. It contains a total of 138 resolution of 4020 x 4892, or
* Reference: [MC-2014]_
* Original resolution (height x width): 4020 x 4892, or 4892 x 4020
* Configuration resolution: 512 x 512 (after rescaling)
-* Split reference: [GAÁL-2020]_
+* Split reference: [GAAL-2020]_
* Protocol ``default``:
* Training samples: 96 (including labels)
@@ -36,7 +36,7 @@ _protocols = [
]
_root_path = bob.extension.rc.get(
- "bob.ip.binseg.MC.datadir", os.path.realpath(os.curdir)
+ "bob.ip.binseg.montgomery.datadir", os.path.realpath(os.curdir)
)
diff --git a/bob/ip/binseg/data/MC/default.json b/bob/ip/binseg/data/montgomery/default.json
similarity index 100%
rename from bob/ip/binseg/data/MC/default.json
rename to bob/ip/binseg/data/montgomery/default.json
diff --git a/bob/ip/binseg/data/Shenzhen/__init__.py b/bob/ip/binseg/data/shenzhen/__init__.py
similarity index 91%
rename from bob/ip/binseg/data/Shenzhen/__init__.py
rename to bob/ip/binseg/data/shenzhen/__init__.py
index 89f9e723de92f92587c929cb8579f30b00469360..689b72ba87cd810ece37a9a8fb7c984c8143f8f1 100644
--- a/bob/ip/binseg/data/Shenzhen/__init__.py
+++ b/bob/ip/binseg/data/shenzhen/__init__.py
@@ -11,7 +11,7 @@ available for 566 of the 662 images.
* Reference: [SHENZHEN-2014]_
* Original resolution (height x width): Approximately 3K x 3K (varies)
* Configuration resolution: 512 x 512 (after rescaling)
-* Split reference: [GAÁL-2020]_
+* Split reference: [GAAL-2020]_
* Protocol ``default``:
* Training samples: 396 (including labels)
@@ -34,7 +34,7 @@ _protocols = [
]
_root_path = bob.extension.rc.get(
- "bob.ip.binseg.Shenzhen.datadir", os.path.realpath(os.curdir)
+ "bob.ip.binseg.shenzhen.datadir", os.path.realpath(os.curdir)
)
@@ -55,4 +55,4 @@ dataset = JSONDataset(
protocols=_protocols, fieldnames=("data", "label"), loader=_loader
)
-"""Shenzhen dataset object"""
+"""Shenzhen CXR dataset object"""
diff --git a/bob/ip/binseg/data/Shenzhen/default.json b/bob/ip/binseg/data/shenzhen/default.json
similarity index 100%
rename from bob/ip/binseg/data/Shenzhen/default.json
rename to bob/ip/binseg/data/shenzhen/default.json
diff --git a/bob/ip/binseg/test/test_jsrt.py b/bob/ip/binseg/test/test_jsrt.py
index e275d81ffc5a10d0cf6e2a45d2a28a8acab3ceef..37f3361406399fbebd70e3b6278e713c5948534b 100644
--- a/bob/ip/binseg/test/test_jsrt.py
+++ b/bob/ip/binseg/test/test_jsrt.py
@@ -7,7 +7,7 @@
import numpy
import pytest
-from ..data.JSRT import dataset
+from ..data.jsrt import dataset
from .utils import count_bw
@@ -32,7 +32,7 @@ def test_protocol_consistency():
assert s.key.startswith("JSRT")
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.JSRT.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.jsrt.datadir")
def test_loading():
image_size = (2048, 2048)
@@ -74,6 +74,6 @@ def test_loading():
del proportions # only to satisfy flake8
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.JSRT.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.jsrt.datadir")
def test_check():
assert dataset.check() == 0
diff --git a/bob/ip/binseg/test/test_mc.py b/bob/ip/binseg/test/test_mc.py
index cfe82bd6a75a4be5567177ffbbeb243fb04b7e89..7df4bd013266e422d9ce53037e40904e9661963b 100644
--- a/bob/ip/binseg/test/test_mc.py
+++ b/bob/ip/binseg/test/test_mc.py
@@ -2,12 +2,12 @@
# coding=utf-8
-"""Tests for Montgomery County"""
+"""Tests for Montgomery County CXR dataset"""
import numpy
import pytest
-from ..data.MC import dataset
+from ..data.montgomery import dataset
from .utils import count_bw
@@ -32,7 +32,7 @@ def test_protocol_consistency():
assert s.key.startswith("MontgomerySet")
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.MC.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.montgomery.datadir")
def test_loading():
image_size_1 = (4892, 4020)
@@ -80,6 +80,6 @@ def test_loading():
del proportions # only to satisfy flake8
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.MC.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.montgomery.datadir")
def test_check():
assert dataset.check() == 0
diff --git a/bob/ip/binseg/test/test_shenzhen.py b/bob/ip/binseg/test/test_shenzhen.py
index d710afda5cd645bd11f0364929cec297a1f9c824..95a1e2c8293dcc829e4d039e907ad2a72ad8d883 100644
--- a/bob/ip/binseg/test/test_shenzhen.py
+++ b/bob/ip/binseg/test/test_shenzhen.py
@@ -2,12 +2,12 @@
# coding=utf-8
-"""Tests for Shenzhen"""
+"""Tests for Shenzhen CXR dataset"""
import numpy
import pytest
-from ..data.Shenzhen import dataset
+from ..data.shenzhen import dataset
from .utils import count_bw
@@ -32,7 +32,7 @@ def test_protocol_consistency():
assert s.key.startswith("ChinaSet_AllFiles")
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.Shenzhen.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.shenzhen.datadir")
def test_loading():
min_image_size = (1130, 948)
@@ -76,6 +76,6 @@ def test_loading():
del proportions # only to satisfy flake8
-@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.Shenzhen.datadir")
+@pytest.mark.skip_if_rc_var_not_set("bob.ip.binseg.shenzhen.datadir")
def test_check():
assert dataset.check() == 0
diff --git a/doc/api.rst b/doc/api.rst
index c37286f1191876b3de0fe8d3be031f721703f662..a97b25b3cd3cd2032896030812f1bd364e7a972e 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -25,6 +25,9 @@ Data Manipulation
Datasets
--------
+Retinography
+============
+
.. autosummary::
:toctree: api/dataset
@@ -38,9 +41,17 @@ Datasets
bob.ip.binseg.data.rimoner3
bob.ip.binseg.data.drionsdb
bob.ip.binseg.data.drhagis
- bob.ip.binseg.data.MC
- bob.ip.binseg.data.JSRT
- bob.ip.binseg.data.Shenzhen
+
+
+Chest X-Ray
+===========
+
+.. autosummary::
+ :toctree: api/dataset
+
+ bob.ip.binseg.data.montgomery
+ bob.ip.binseg.data.jsrt
+ bob.ip.binseg.data.shenzhen
Engines
@@ -193,11 +204,11 @@ Datasets
bob.ip.binseg.configs.datasets.drhagis.default
- bob.ip.binseg.configs.datasets.MC.default
+ bob.ip.binseg.configs.datasets.montgomery.default
- bob.ip.binseg.configs.datasets.JSRT.default
+ bob.ip.binseg.configs.datasets.jsrt.default
- bob.ip.binseg.configs.datasets.Shenzhen.default
- bob.ip.binseg.configs.datasets.Shenzhen.default_256
+ bob.ip.binseg.configs.datasets.shenzhen.default
+ bob.ip.binseg.configs.datasets.shenzhen.default_256
.. include:: links.rst
diff --git a/doc/conf.py b/doc/conf.py
index 7138e12c84844e7a7d8360bb5f029a88c699a318..e2da7c0ce756c6cdee4cc702b6000140696667a8 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -25,6 +25,7 @@ extensions = [
"sphinx.ext.viewcode",
"sphinx.ext.mathjax",
"sphinxcontrib.programoutput",
+ "sphinx_rtd_theme",
# 'matplotlib.sphinxext.plot_directive'
]
diff --git a/doc/datasets.rst b/doc/datasets.rst
index 42be623c43352e17ddeb5f84839f487d9c4b8569..f5717e4a72f18f19346470cad5aa511c0093d4c4 100644
--- a/doc/datasets.rst
+++ b/doc/datasets.rst
@@ -12,7 +12,11 @@ can be downloaded. We include the reference of the data split protocols used
to generate iterators for training and testing.
-.. list-table:: Supported Datasets (``*`` provided within this package)
+Retinography
+------------
+
+
+.. list-table:: Supported Retinography Datasets (``*``: provided within this package)
* - Dataset
- Reference
@@ -29,8 +33,8 @@ to generate iterators for training and testing.
- [DRIVE-2004]_
- 584 x 565
- 40
- - x
- - x
+ - ``x``
+ - ``x``
-
-
- [DRIVE-2004]_
@@ -40,8 +44,8 @@ to generate iterators for training and testing.
- [STARE-2000]_
- 605 x 700
- 20
- - *
- - x
+ - ``*``
+ - ``x``
-
-
- [MANINIS-2016]_
@@ -51,8 +55,8 @@ to generate iterators for training and testing.
- [CHASEDB1-2012]_
- 960 x 999
- 28
- - *
- - x
+ - ``*``
+ - ``x``
-
-
- [CHASEDB1-2012]_
@@ -62,8 +66,8 @@ to generate iterators for training and testing.
- [HRF-2013]_
- 2336 x 3504
- 45
- - x
- - x
+ - ``x``
+ - ``x``
-
-
- [ORLANDO-2017]_
@@ -73,9 +77,9 @@ to generate iterators for training and testing.
- [IOSTAR-2016]_
- 1024 x 1024
- 30
- - x
- - x
- - x
+ - ``x``
+ - ``x``
+ - ``x``
-
- [MEYER-2017]_
- 20
@@ -86,7 +90,7 @@ to generate iterators for training and testing.
- 110
-
-
- - x
+ - ``x``
-
- [MANINIS-2016]_
- 60
@@ -97,8 +101,8 @@ to generate iterators for training and testing.
- 159
-
-
- - x
- - x
+ - ``x``
+ - ``x``
- [MANINIS-2016]_
- 99
- 60
@@ -108,8 +112,8 @@ to generate iterators for training and testing.
- 101
-
-
- - x
- - x
+ - ``x``
+ - ``x``
- [DRISHTIGS1-2014]_
- 50
- 51
@@ -119,8 +123,8 @@ to generate iterators for training and testing.
- 1200
-
-
- - x
- - x
+ - ``x``
+ - ``x``
- [REFUGE-2018]_
- 400 (+400)
- 400
@@ -128,44 +132,50 @@ to generate iterators for training and testing.
- [DRHAGIS-2017]_
- Varying
- 39
- - x
- - x
+ - ``x``
+ - ``x``
-
-
- [DRHAGIS-2017]_
- 19
- 20
- * - MC_
+
+
+X-Ray
+-----
+
+.. list-table:: Supported X-Ray Datasets
+
+ * - Dataset
+ - Reference
+ - H x W
+ - Samples
+ - Mask
+ - Split Reference
+ - Train
+ - Test
+ * - `Montgomery County`_
- [MC-2014]_
- 4020 x 4892, or 4892 x 4020
- 138
- - x
- -
- -
- -
- - [GAÁL-2020]_
+ - ``x``
+ - [GAAL-2020]_
- 96 (+14)
- 28
* - JSRT_
- [JSRT-2000]_
- 2048 x 2048
- 247
- - x
- -
- -
- -
- - [GAÁL-2020]_
+ - ``x``
+ - [GAAL-2020]_
- 172 (+25)
- 50
- * - SHENZHEN_
+ * - Shenzhen_
- [SHENZHEN-2014]_
- Varying
- 662
- -
- -
- -
- -
- - [GAÁL-2020]_
+ - ``x``
+ - [GAAL-2020]_
- 396 (+56)
- 114
diff --git a/doc/experiment.rst b/doc/experiment.rst
index 271d1290723588552bb33c3182d4042779d2fdd9..7a9bc66cafc11e7803f6cab85b631facfa45550a 100644
--- a/doc/experiment.rst
+++ b/doc/experiment.rst
@@ -12,9 +12,10 @@ evaluation and comparison. After running, you will be able to find results
from model fitting, prediction, evaluation and comparison under a single output
directory.
-For example, to train a Mobile V2 U-Net architecture on the STARE dataset,
-evaluate both train and test set performances, output prediction maps and
-overlay analysis, together with a performance curve, run the following:
+For example, to train a Mobile V2 U-Net architecture on the STARE dataset
+(optic vessel segmentation), evaluate both train and test set performances,
+output prediction maps and overlay analysis, together with a performance curve,
+run the following:
.. code-block:: sh
@@ -41,60 +42,15 @@ happy with the result, use it in place of one of our datasets:
$ bob binseg experiment ... mydataset.py ...
-Baseline Benchmarks
-===================
+Changing defaults
+=================
-The following table describes recommended batch sizes for 24Gb of RAM GPU
-card, for supervised training of baselines. Use it like this:
-
-.. code-block:: sh
-
- # change <model> and <dataset> by one of items bellow
- $ bob binseg experiment -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
- # check results in the "results" folder
-
-.. list-table::
-
- * - **Models / Datasets**
- - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
- - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
- - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
- - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
- - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`
- * - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
- - 4
- - 2
- - 2
- - 2
- - 1
- * - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
- - 8
- - 4
- - 4
- - 4
- - 1
- * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
- - 8
- - 5
- - 4
- - 4
- - 1
- * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
- - 16
- - 6
- - 6
- - 6
- - 1
-
-
-.. tip::
-
- Instead of the default configurations, you can pass the full path of your
- customized dataset and model files. You may :ref:`copy any of the existing
- configuration resources <bob.ip.binseg.cli.config.copy>` and change them
- locally. Once you're happy, you may use the newly created files directly on
- your command line. For example, suppose you wanted to slightly change the
- DRIVE pre-processing pipeline. You could do the following:
+We provide a large set of preset configurations to build models from known
+datasets. You can :ref:`copy any of the existing configuration resources
+<bob.ip.binseg.cli.config.copy>` and edit to build your own customized version.
+Once you're happy, you may use the newly created files directly on your command
+line. For example, suppose you wanted to slightly change the DRIVE
+pre-processing pipeline. You could do the following:
.. code-block:: bash
@@ -105,88 +61,20 @@ card, for supervised training of baselines. Use it like this:
.. _bob.ip.binseg.gridtk-tip:
-.. tip::
-
- If you are at Idiap, you may install the package ``gridtk`` (``conda install
- gridtk``) on your environment, and submit the job like this:
-
- .. code-block:: sh
-
- $ jman submit --queue=gpu --memory=24G --name=myjob -- bob binseg train --device='cuda:0' ... #paste the rest of the command-line
+Running at Idiap's SGE grid
+===========================
-.. _bob.ip.binseg.baseline-script:
-
-The :download:`following shell script <scripts/baselines.sh>` can run the
-various baselines described above and place results in a single directory:
-
-.. literalinclude:: scripts/baselines.sh
- :language: bash
-
-You will find results obtained running these baselines :ref:`further in this
-guide <bob.ip.binseg.results.baselines>`.
-
-
-Combined Vessel Dataset (COVD)
-==============================
-
-The following table describes recommended batch sizes for 24Gb of RAM GPU card,
-for supervised training of COVD- systems. Use it like this:
+If you are at Idiap, you may install the package ``gridtk`` (``conda install
+gridtk``) on your environment, and submit the job like this:
.. code-block:: sh
- # change <model> and <dataset> by one of items bellow
- $ bob binseg experiment -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
-
-.. list-table::
-
- * - **Models / Datasets**
- - :py:mod:`drive-covd <bob.ip.binseg.configs.datasets.drive.covd>`
- - :py:mod:`stare-covd <bob.ip.binseg.configs.datasets.stare.covd>`
- - :py:mod:`chasedb1-covd <bob.ip.binseg.configs.datasets.chasedb1.covd>`
- - :py:mod:`iostar-vessel-covd <bob.ip.binseg.configs.datasets.iostar.covd>`
- - :py:mod:`hrf-covd <bob.ip.binseg.configs.datasets.hrf.covd>`
- * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
- - 4
- - 4
- - 2
- - 2
- - 2
- * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
- - 8
- - 4
- - 4
- - 4
- - 4
-
-
-Combined Vessel Dataset (COVD) and Semi-Supervised Learning (SSL)
-=================================================================
-
-The following table describes recommended batch sizes for 24Gb of RAM GPU
-card, for semi-supervised learning of COVD- systems. Use it like this:
+ $ jman submit --queue=gpu --memory=24G --name=myjob -- bob binseg train --device='cuda:0' ... #paste the rest of the command-line
-.. code-block:: sh
+:download:`This bash-script function <scripts/functions.sh>` can be of help
+when switching between local and SGE-based running. Just copy and source this
+file, then call the function ``run`` as many times as required to benchmark
+your task.
- # change <model> and <dataset> by one of items bellow
- $ bob binseg train -vv --ssl <model> <dataset> --batch-size=<see-table> --device="cuda:0"
-
-.. list-table::
-
- * - **Models / Datasets**
- - :py:mod:`drive-ssl <bob.ip.binseg.configs.datasets.drive.ssl>`
- - :py:mod:`stare-ssl <bob.ip.binseg.configs.datasets.stare.ssl>`
- - :py:mod:`chasedb1-ssl <bob.ip.binseg.configs.datasets.chasedb1.ssl>`
- - :py:mod:`iostar-vessel-ssl <bob.ip.binseg.configs.datasets.iostar.ssl>`
- - :py:mod:`hrf-ssl <bob.ip.binseg.configs.datasets.hrf.ssl>`
- * - :py:mod:`driu-ssl <bob.ip.binseg.configs.models.driu_ssl>` / :py:mod:`driu-bn-ssl <bob.ip.binseg.configs.models.driu_bn_ssl>`
- - 4
- - 4
- - 2
- - 1
- - 1
- * - :py:mod:`m2unet-ssl <bob.ip.binseg.configs.models.m2unet_ssl>`
- - 4
- - 4
- - 2
- - 2
- - 2
+ .. literalinclude:: scripts/functions.sh
+ :language: bash
diff --git a/doc/framework.dot b/doc/framework.dot
index 50bfce5a2ff0030ee75c0138efd884c4e8f3228b..ea2e51ab239f4dbf9e1d01ff3659df1391213221 100644
--- a/doc/framework.dot
+++ b/doc/framework.dot
@@ -49,7 +49,7 @@ digraph framework {
];
{rank = max; figure; table; }
- dataset:train -> train [headlabel="sample + label",labelangle=30];
+ dataset:train -> train [headlabel="sample + label [+ mask]",labelangle=30];
dataset:test -> predict [headlabel="sample",labelangle=30];
train -> predict [headlabel="model"];
dataset:test -> evaluate [headlabel="label"];
diff --git a/doc/index.rst b/doc/index.rst
index e55d764d38d7ca1527092f7676e4d613b0239ccd..1f26c069e0121ba3f9f08ce60756d0729f1995c4 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -7,32 +7,30 @@
===============================================
Package to benchmark and evaluate a range of neural network architectures for
-binary segmentation tasks on 2D Eye Fundus Images (2DFI). It is build using
-PyTorch.
+binary segmentation tasks. It is build using PyTorch.
-Please use the BibTeX reference below to cite this work:
+Please at least use the BibTeX references below to cite this work:
.. code:: bibtex
- @misc{laibacher_anjos_2019,
- title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Retinography},
- author = {Tim Laibacher and Andr\'e Anjos},
- year = {2019},
- eprint = {1909.03856},
- archivePrefix = {arXiv},
- primaryClass = {cs.CV},
- url = {https://arxiv.org/abs/1909.03856},
+ @inproceedings{renzo_2021,
+ title = {Development of a lung segmentation algorithm for analog imaged chest X-Ray: preliminary results},
+ author = {Matheus A. Renzo and Nat\'{a}lia Fernandez and Andr\'e Baceti and Natanael Nunes de Moura Junior and Andr\'e Anjos},
+ month = {10},
+ booktitle = {XV Brazilian Congress on Computational Intelligence},
+ year = {2021},
+ url = {https://publications.idiap.ch/index.php/publications/show/4649},
}
-
-Additional Material
--------------------
-
-The additional material referred to in the paper can be found under
-:ref:`bob.ip.binseg.results` and :download:`here <additionalresults.pdf>`
-
-
-.. todolist::
+ @misc{laibacher_2019,
+ title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Retinography},
+ author = {Tim Laibacher and Andr\'e Anjos},
+ year = {2019},
+ eprint = {1909.03856},
+ archivePrefix = {arXiv},
+ primaryClass = {cs.CV},
+ url = {https://arxiv.org/abs/1909.03856},
+ }
User Guide
diff --git a/doc/links.rst b/doc/links.rst
index 3db77da20c2c46016f4cc83a76d9eaa8519f4c66..44fc48978df3ca6b4a6eb186c53a0766a099897b 100644
--- a/doc/links.rst
+++ b/doc/links.rst
@@ -24,9 +24,9 @@
.. _drishti-gs1: http://cvit.iiit.ac.in/projects/mip/drishti-gs/mip-dataset2/Home.php
.. _refuge: https://refuge.grand-challenge.org/Details/
.. _drhagis: https://personalpages.manchester.ac.uk/staff/niall.p.mcloughlin/
-.. _MC: https://openi.nlm.nih.gov/faq#faq-tb-coll
-.. _JSRT: http://db.jsrt.or.jp/eng.php
-.. _Shenzhen: https://openi.nlm.nih.gov/faq#faq-tb-coll
+.. _montgomery county: https://openi.nlm.nih.gov/faq#faq-tb-coll
+.. _jsrt: http://db.jsrt.or.jp/eng.php
+.. _shenzhen: https://openi.nlm.nih.gov/faq#faq-tb-coll
.. Software Tools
diff --git a/doc/references.rst b/doc/references.rst
index d3cfd3972f7f815415ef670ab53fadf08fceb72c..ccd7af77f91e89cf2d413f59dd34fcf18cb78bab 100644
--- a/doc/references.rst
+++ b/doc/references.rst
@@ -101,7 +101,7 @@
**MobileNetV2: Inverted Residuals and Linear Bottlenecks**, 2018.
https://arxiv.org/abs/1801.04381
-.. [DAVIS-2006] *J. Davis and M. Goadrich*, **The relationship between
+.. .. [DAVIS-2006] *J. Davis and M. Goadrich*, **The relationship between
Precision-Recall and ROC curves**. 23rd international conference on Machine
learning (ICML’06), 2006. https://doi.org/10.1145/1143844.1143874
@@ -139,6 +139,6 @@
pulmonary diseases.**, Quantitative imaging in medicine and surgery. 2014.
https://doi:10.3978/j.issn.2223-4292.2014.11.20
-.. [GAÁL-2020] *G. Gaál, B. Maga, A. Lukács*, **Attention U-Net Based
+.. [GAAL-2020] *G. Gaál, B. Maga, A. Lukács*, **Attention U-Net Based
Adversarial Architectures for Chest X-ray Lung Segmentation.**, 2020.
https://arxiv.org/abs/2003.10304v1
diff --git a/doc/results/baselines/chasedb1.pdf b/doc/results/baselines/chasedb1.pdf
deleted file mode 100644
index d9418224cffb8725c569e268885f1dc559407e4b..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/chasedb1.pdf and /dev/null differ
diff --git a/doc/results/baselines/chasedb1.png b/doc/results/baselines/chasedb1.png
deleted file mode 100644
index ae1ebe8b05017c3ca49d662830252c612d9f16e4..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/chasedb1.png and /dev/null differ
diff --git a/doc/results/baselines/drive.pdf b/doc/results/baselines/drive.pdf
deleted file mode 100644
index d54253ef754323baf7ebe156ce5016d72e343489..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/drive.pdf and /dev/null differ
diff --git a/doc/results/baselines/drive.png b/doc/results/baselines/drive.png
deleted file mode 100644
index d0247edfd89325fd028aa67c6328ad7f269c57b4..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/drive.png and /dev/null differ
diff --git a/doc/results/baselines/hrf-fullres.pdf b/doc/results/baselines/hrf-fullres.pdf
deleted file mode 100644
index a3473f2f3a685fa27eed05d9c7c8c23b517358ea..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/hrf-fullres.pdf and /dev/null differ
diff --git a/doc/results/baselines/hrf-fullres.png b/doc/results/baselines/hrf-fullres.png
deleted file mode 100644
index f33a948e00f2bcf79be8afcb8b32bdfbc4775ae8..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/hrf-fullres.png and /dev/null differ
diff --git a/doc/results/baselines/hrf.pdf b/doc/results/baselines/hrf.pdf
deleted file mode 100644
index a8e5d7342338aa09971efb47aa801d268c49cc9e..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/hrf.pdf and /dev/null differ
diff --git a/doc/results/baselines/hrf.png b/doc/results/baselines/hrf.png
deleted file mode 100644
index b1e006be7647212abd6fb53c0c213d097fc416a3..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/hrf.png and /dev/null differ
diff --git a/doc/results/baselines/index.rst b/doc/results/baselines/index.rst
index 70bb198e2689e37cdb3b5c7d7757709c424166af..94416e1221e5065325d33e7cff39fc140d42f2c1 100644
--- a/doc/results/baselines/index.rst
+++ b/doc/results/baselines/index.rst
@@ -6,9 +6,6 @@
Baseline Results
===================
-F1 Scores (micro-level)
------------------------
-
* Benchmark results for models: DRIU, HED, M2U-Net, U-Net, and Little W-Net.
* Models are trained and tested on the same dataset (**numbers in bold**
indicate approximate number of parameters per model). DRIU, HED, M2U-Net and
@@ -35,153 +32,16 @@ F1 Scores (micro-level)
analyze`` providing the model URL as ``--weight`` parameter.
* For comparison purposes, we provide "second-annotator" performances on the
same test set, where available.
-* :ref:`Our baseline script <bob.ip.binseg.baseline-script>` was used to
- generate the results displayed here.
-* HRF models were trained using half the full resolution (1168x1648)
-
-
-.. list-table::
- :header-rows: 2
-
- * -
- -
- - :py:mod:`driu <bob.ip.binseg.configs.models.driu>`
- - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
- - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
- - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
- - lwnet
- * - Dataset
- - 2nd. Annot.
- - 15M
- - 14.7M
- - 550k
- - 25.8M
- - 68k
- * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
- - 0.788 (0.021)
- - `0.821 (0.014) <baselines_driu_drive_>`_
- - `0.813 (0.016) <baselines_hed_drive_>`_
- - `0.802 (0.014) <baselines_m2unet_drive_>`_
- - `0.825 (0.015) <baselines_unet_drive_>`_
- - 0.828
- * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
- - 0.759 (0.028)
- - `0.828 (0.039) <baselines_driu_stare_>`_
- - `0.815 (0.047) <baselines_hed_stare_>`_
- - `0.818 (0.035) <baselines_m2unet_stare_>`_
- - `0.828 (0.050) <baselines_unet_stare_>`_
- - 0.839
- * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
- - 0.768 (0.023)
- - `0.812 (0.018) <baselines_driu_chase_>`_
- - `0.806 (0.020) <baselines_hed_chase_>`_
- - `0.798 (0.018) <baselines_m2unet_chase_>`_
- - `0.807 (0.017) <baselines_unet_chase_>`_
- - 0.820
- * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (1168x1648)
- -
- - `0.808 (0.038) <baselines_driu_hrf_>`_
- - `0.803 (0.040) <baselines_hed_hrf_>`_
- - `0.796 (0.048) <baselines_m2unet_hrf_>`_
- - `0.811 (0.039) <baselines_unet_hrf_>`_
- - 0.814
- * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (2336x3296)
- -
- - `0.722 (0.073) <baselines_driu_hrf_>`_
- - `0.703 (0.090) <baselines_hed_hrf_>`_
- - `0.713 (0.143) <baselines_m2unet_hrf_>`_
- - `0.756 (0.051) <baselines_unet_hrf_>`_
- - 0.744
- * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
- -
- - `0.825 (0.020) <baselines_driu_iostar_>`_
- - `0.827 (0.020) <baselines_hed_iostar_>`_
- - `0.820 (0.018) <baselines_m2unet_iostar_>`_
- - `0.818 (0.020) <baselines_unet_iostar_>`_
- - 0.832
-
-Precision-Recall (PR) Curves
-----------------------------
-
-Next, you will find the PR plots showing confidence intervals, for the various
-models explored, on a per dataset arrangement. All curves correspond to test
-set performances. Single performance figures (F1-micro scores) correspond to
-its average value across all test set images, for a fixed threshold set to
-``0.5``, and using 1000 points for curve calculation.
-
-.. tip:: **Curve Intepretation**
-
- PR curves behave differently than traditional ROC curves (using Specificity
- versus Sensitivity) with respect to the overall shape. You may have a look
- at [DAVIS-2006]_ for details on the relationship between PR and ROC curves.
- For example, PR curves are not guaranteed to be monotonically increasing or
- decreasing with the scanned thresholds.
-
- Each evaluated threshold in a combination of trained models and datasets is
- represented by a point in each curve. Points are linearly interpolated to
- created a line. For each evaluated threshold and every trained model and
- dataset, we assume that the standard deviation on both precision and recall
- estimation represent good proxies for the uncertainty around that point. We
- therefore plot a transparent ellipse centered around each evaluated point in
- which the width corresponds to twice the recall standard deviation and the
- height, twice the precision standard deviation.
-
-
-.. list-table::
-
- * - .. figure:: drive.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for drive datasets
-
- :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`: PR curve and F1 scores at T=0.5 (:download:`pdf <drive.pdf>`)
- - .. figure:: stare.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for stare datasets
-
- :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`: PR curve and F1 scores at T=0.5 (:download:`pdf <stare.pdf>`)
- * - .. figure:: chasedb1.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for chasedb1 datasets
-
- :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`: PR curve and F1 scores at T=0.5 (:download:`pdf <chasedb1.pdf>`)
- - .. figure:: hrf.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for hrf datasets (matching training resolution: 1168x1648)
-
- :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (1168x1648): PR curve and F1 scores at T=0.5 (:download:`pdf <hrf.pdf>`)
- * - .. figure:: iostar-vessel.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for iostar-vessel datasets
-
- :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`: PR curve and F1 scores at T=0.5 (:download:`pdf <iostar-vessel.pdf>`)
- - .. figure:: hrf-fullres.png
- :align: center
- :scale: 50%
- :alt: Model comparisons for hrf datasets (double training resolution: 2336x3296)
- :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (2336x3296): PR curve and F1 scores at T=0.5 (:download:`pdf <hrf-fullres.pdf>`)
+Tasks
+-----
-Remarks
--------
+.. toctree::
+ :maxdepth: 1
-* There seems to be no clear winner as confidence intervals based on the
- standard deviation overlap substantially between the different models, and
- across different datasets.
-* There seems to be almost no effect on the number of parameters on
- performance. U-Net, the largest model, is not a clear winner through all
- baseline benchmarks
-* Where second annotator labels exist, model performance and variability seems
- on par with such annotations. One possible exception is for CHASE-DB1, where
- models show consistently less variability than the second annotator.
- Unfortunately, this is not conclusive.
-* Training at half resolution for HRF shows a small loss in performance (10 to
- 15%) when the high-resolution version is used as evaluation set.
+ vessel
+ lung
.. include:: ../../links.rst
diff --git a/doc/results/baselines/iostar-vessel.pdf b/doc/results/baselines/iostar-vessel.pdf
deleted file mode 100644
index 0c9f08d12f8a919dd0c381fafc4fc52fa0e50101..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/iostar-vessel.pdf and /dev/null differ
diff --git a/doc/results/baselines/iostar-vessel.png b/doc/results/baselines/iostar-vessel.png
deleted file mode 100644
index a670c64c1e50c2f68b4f1ec95df85050d25887ab..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/iostar-vessel.png and /dev/null differ
diff --git a/doc/results/baselines/lung.rst b/doc/results/baselines/lung.rst
new file mode 100644
index 0000000000000000000000000000000000000000..47889687af36195f24055e135ece9c3ea2e462d3
--- /dev/null
+++ b/doc/results/baselines/lung.rst
@@ -0,0 +1,29 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.baselines.lung:
+
+=============================================
+ Lung Segmentation from Frontal Chest X-Rays
+=============================================
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - :py:mod:`lwnet <bob.ip.binseg.configs.models.lwnet>`
+ * - Dataset
+ - 550k
+ - 68k
+ * - :py:mod:`montgomery <bob.ip.binseg.configs.datasets.montgomery.default>`
+ - 0.982
+ - 0.975
+ * - :py:mod:`jsrt <bob.ip.binseg.configs.datasets.jsrt.default>`
+ - 0.982
+ - 0.977
+ * - :py:mod:`shenzhen <bob.ip.binseg.configs.datasets.shenzhen.default>`
+ - 0.955
+ - 0.952
+
+
+.. include:: ../../links.rst
diff --git a/doc/results/baselines/stare.pdf b/doc/results/baselines/stare.pdf
deleted file mode 100644
index 641e6e2edd9da985ecf64067c9ddf97941a271ee..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/stare.pdf and /dev/null differ
diff --git a/doc/results/baselines/stare.png b/doc/results/baselines/stare.png
deleted file mode 100644
index 76fe040196eaf04f64276f3fa5b0a5c2d2f4ac27..0000000000000000000000000000000000000000
Binary files a/doc/results/baselines/stare.png and /dev/null differ
diff --git a/doc/results/baselines/vessel.rst b/doc/results/baselines/vessel.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d7456e21569e3c3beff133a228efc54e2c7484b0
--- /dev/null
+++ b/doc/results/baselines/vessel.rst
@@ -0,0 +1,112 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.baselines.vessel:
+
+==============================================
+ Retinal Vessel Segmentation for Retinography
+==============================================
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ -
+ - :py:mod:`driu <bob.ip.binseg.configs.models.driu>`
+ - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
+ - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
+ - :py:mod:`lwnet <bob.ip.binseg.configs.models.lwnet>`
+ * - Dataset
+ - 2nd. Annot.
+ - 15M
+ - 14.7M
+ - 550k
+ - 25.8M
+ - 68k
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
+ - 0.788 (0.021)
+ - `0.821 (0.014) <baselines_driu_drive_>`_
+ - `0.813 (0.016) <baselines_hed_drive_>`_
+ - `0.802 (0.014) <baselines_m2unet_drive_>`_
+ - `0.825 (0.015) <baselines_unet_drive_>`_
+ - 0.828
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
+ - 0.759 (0.028)
+ - `0.828 (0.039) <baselines_driu_stare_>`_
+ - `0.815 (0.047) <baselines_hed_stare_>`_
+ - `0.818 (0.035) <baselines_m2unet_stare_>`_
+ - `0.828 (0.050) <baselines_unet_stare_>`_
+ - 0.839
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
+ - 0.768 (0.023)
+ - `0.812 (0.018) <baselines_driu_chase_>`_
+ - `0.806 (0.020) <baselines_hed_chase_>`_
+ - `0.798 (0.018) <baselines_m2unet_chase_>`_
+ - `0.807 (0.017) <baselines_unet_chase_>`_
+ - 0.820
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (1168x1648)
+ -
+ - `0.808 (0.038) <baselines_driu_hrf_>`_
+ - `0.803 (0.040) <baselines_hed_hrf_>`_
+ - `0.796 (0.048) <baselines_m2unet_hrf_>`_
+ - `0.811 (0.039) <baselines_unet_hrf_>`_
+ - 0.814
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (2336x3296)
+ -
+ - `0.722 (0.073) <baselines_driu_hrf_>`_
+ - `0.703 (0.090) <baselines_hed_hrf_>`_
+ - `0.713 (0.143) <baselines_m2unet_hrf_>`_
+ - `0.756 (0.051) <baselines_unet_hrf_>`_
+ - 0.744
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
+ -
+ - `0.825 (0.020) <baselines_driu_iostar_>`_
+ - `0.827 (0.020) <baselines_hed_iostar_>`_
+ - `0.820 (0.018) <baselines_m2unet_iostar_>`_
+ - `0.818 (0.020) <baselines_unet_iostar_>`_
+ - 0.832
+
+
+Notes
+-----
+
+* HRF models were trained using half the full resolution (1168x1648)
+* The following table describes recommended batch sizes for 24Gb of RAM GPU
+ card:
+
+ .. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
+ - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
+ - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
+ - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
+ - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`
+ * - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
+ - 4
+ - 2
+ - 2
+ - 2
+ - 1
+ * - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
+ - 8
+ - 4
+ - 4
+ - 4
+ - 1
+ * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
+ - 8
+ - 5
+ - 4
+ - 4
+ - 1
+ * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - 16
+ - 6
+ - 6
+ - 6
+ - 1
+
+
+.. include:: ../../links.rst
diff --git a/doc/results/cod/index.rst b/doc/results/cod/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3c4fbf5690b2579dc408572830d4c3bea4f149a7
--- /dev/null
+++ b/doc/results/cod/index.rst
@@ -0,0 +1,33 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.cod:
+
+================================
+ Combined Dataset (COD) Results
+================================
+
+* Models are trained on a COD **excluding** the target dataset, and tested on
+ the target dataset (**numbers in bold** indicate number of parameters per
+ model). Models are trained for a fixed number of 1000 epochs, with a
+ learning rate of 0.001 until epoch 900 and then 0.0001 until the end of the
+ training.
+* Database and model resource configuration links (table top row and left
+ column) are linked to the originating configuration files used to obtain
+ these results.
+* Single performance numbers correspond to *a priori* performance indicators,
+ where the threshold is previously selected on the training set (COD
+ excluding the target dataset)
+* You can cross check the analysis numbers provided in this table by
+ downloading this software package, the raw data, and running ``bob binseg
+ analyze`` providing the model URL as ``--weight`` parameter.
+* For comparison purposes, we provide "second-annotator" performances on the
+ same test set, where available.
+
+
+.. toctree::
+ :maxdepth: 1
+
+ vessel
+
+
+.. include:: ../../links.rst
diff --git a/doc/results/covd/index.rst b/doc/results/cod/vessel.rst
similarity index 55%
rename from doc/results/covd/index.rst
rename to doc/results/cod/vessel.rst
index ad61be21bb40b34d922078c7bb341f7cd624b652..4b27eb958761e6684cf8321e67a1b0c4386be563 100644
--- a/doc/results/covd/index.rst
+++ b/doc/results/cod/vessel.rst
@@ -1,34 +1,10 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.results.covd:
+.. _bob.ip.binseg.results.cod.vessel:
-========================================
- Combined Vessel Dataset (COVD) Results
-========================================
-
-
-F1 Scores (micro-level)
------------------------
-
-* Benchmark results for models: DRIU, HED, M2U-Net and U-Net.
-* Models are trained on a COVD **excluding** the target dataset, and tested on
- the target dataset (**numbers in bold** indicate number of parameters per
- model). Models are trained for a fixed number of 1000 epochs, with a
- learning rate of 0.001 until epoch 900 and then 0.0001 until the end of the
- training.
-* Database and model resource configuration links (table top row and left
- column) are linked to the originating configuration files used to obtain
- these results.
-* Check `our paper`_ for details on the calculation of the F1 Score and standard
- deviations (in parentheses).
-* Single performance numbers correspond to *a priori* performance indicators,
- where the threshold is previously selected on the training set (COVD
- excluding the target dataset)
-* You can cross check the analysis numbers provided in this table by
- downloading this software package, the raw data, and running ``bob binseg
- analyze`` providing the model URL as ``--weight`` parameter.
-* For comparison purposes, we provide "second-annotator" performances on the
- same test set, where available.
+==============================================
+ Retinal Vessel Segmentation for Retinography
+==============================================
.. list-table::
@@ -78,4 +54,37 @@ F1 Scores (micro-level)
- `0.783 (0.019) <covd_unet_iostar_>`_
+Notes
+-----
+
+* The following table describes recommended batch sizes for 24Gb of RAM GPU
+ card, for supervised training of COD-systems:
+
+ .. code-block:: sh
+
+ # change <model> and <dataset> by one of items bellow
+ $ bob binseg experiment -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+ .. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive-covd <bob.ip.binseg.configs.datasets.drive.covd>`
+ - :py:mod:`stare-covd <bob.ip.binseg.configs.datasets.stare.covd>`
+ - :py:mod:`chasedb1-covd <bob.ip.binseg.configs.datasets.chasedb1.covd>`
+ - :py:mod:`iostar-vessel-covd <bob.ip.binseg.configs.datasets.iostar.covd>`
+ - :py:mod:`hrf-covd <bob.ip.binseg.configs.datasets.hrf.covd>`
+ * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
+ - 4
+ - 4
+ - 2
+ - 2
+ - 2
+ * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - 8
+ - 4
+ - 4
+ - 4
+ - 4
+
+
.. include:: ../../links.rst
diff --git a/doc/results/index.rst b/doc/results/index.rst
index ed6e9f05910d40d7061447a707e290baca17affa..055bc49e18599f24661bf9fc8af9d24abaffa182 100644
--- a/doc/results/index.rst
+++ b/doc/results/index.rst
@@ -9,7 +9,7 @@
This section summarizes results that can be obtained with this package, and
were presented in our paper. We organize the result section in two parts, for
covering baseline results (training and testing on the same dataset) and
-results using our Combined Vessel Dataset minus target dataset (COVD-) training
+results using our Combined Vessel Dataset minus target dataset (COD-) training
strategy.
.. toctree::
@@ -17,7 +17,7 @@ strategy.
baselines/index
xtest/index
- covd/index
+ cod/index
old/index
diff --git a/doc/results/old/index.rst b/doc/results/old/index.rst
index 5577a4dcba6ccab624823cee08301db6aa7ae2bf..f87bcb23f9f39a1c8476fc515a4e6d34b2b7319d 100644
--- a/doc/results/old/index.rst
+++ b/doc/results/old/index.rst
@@ -7,23 +7,52 @@
This section is outdated and needs re-factoring.
-============================
- COVD- and COVD-SLL Results
-============================
+=========================================
+ COVD- and COVD-SLL Results (Deprecated)
+=========================================
In addition to the M2U-Net architecture, we also evaluated the larger DRIU
network and a variation of it that contains batch normalization (DRIU+BN) on
COVD- (Combined Vessel Dataset from all training data minus target test set)
-and COVD-SSL (COVD- and Semi-Supervised Learning). Perhaps surprisingly, for
-the majority of combinations, the performance of the DRIU variants are roughly
-equal or worse to the ones obtained with the much smaller M2U-Net. We
-anticipate that one reason for this could be overparameterization of large
-VGG-16 models that are pretrained on ImageNet.
+and SSL (Semi-Supervised Learning). Perhaps surprisingly, for the majority of
+combinations, the performance of the DRIU variants are roughly equal or worse
+to the ones obtained with the much smaller M2U-Net. We anticipate that one
+reason for this could be overparameterization of large VGG-16 models that are
+pretrained on ImageNet.
F1 Scores
---------
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for semi-supervised learning of COVD- systems. Use it like this:
+
+.. code-block:: sh
+
+ # change <model> and <dataset> by one of items bellow
+ $ bob binseg train -vv --ssl <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive-ssl <bob.ip.binseg.configs.datasets.drive.ssl>`
+ - :py:mod:`stare-ssl <bob.ip.binseg.configs.datasets.stare.ssl>`
+ - :py:mod:`chasedb1-ssl <bob.ip.binseg.configs.datasets.chasedb1.ssl>`
+ - :py:mod:`iostar-vessel-ssl <bob.ip.binseg.configs.datasets.iostar.ssl>`
+ - :py:mod:`hrf-ssl <bob.ip.binseg.configs.datasets.hrf.ssl>`
+ * - :py:mod:`driu-ssl <bob.ip.binseg.configs.models.driu_ssl>` / :py:mod:`driu-bn-ssl <bob.ip.binseg.configs.models.driu_bn_ssl>`
+ - 4
+ - 4
+ - 2
+ - 1
+ - 1
+ * - :py:mod:`m2unet-ssl <bob.ip.binseg.configs.models.m2unet_ssl>`
+ - 4
+ - 4
+ - 2
+ - 2
+ - 2
+
Comparison of F1 Scores (micro-level and standard deviation) of DRIU and
M2U-Net on COVD- and COVD-SSL. Standard deviation across test-images in
brackets.
@@ -77,46 +106,4 @@ brackets.
- `0.785 (0.018) <m2unet_covd-iostar_ssl.pth>`_
-M2U-Net Precision vs. Recall Curves
------------------------------------
-
-Precision vs. recall curves for each evaluated dataset. Note that here the
-F1-score is calculated on a macro level (see paper for more details).
-
-.. figure:: pr_CHASEDB1.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- CHASE_DB1: Precision vs Recall curve and F1 scores
-
-.. figure:: pr_DRIVE.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- DRIVE: Precision vs Recall curve and F1 scores
-
-.. figure:: pr_HRF.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- HRF: Precision vs Recall curve and F1 scores
-
-.. figure:: pr_IOSTARVESSEL.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- IOSTAR: Precision vs Recall curve and F1 scores
-
-.. figure:: pr_STARE.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- STARE: Precision vs Recall curve and F1 scores
-
-
.. include:: ../../links.rst
diff --git a/doc/results/old/pr_CHASEDB1.png b/doc/results/old/pr_CHASEDB1.png
deleted file mode 100644
index 923a4af3445926fe46cb3ac58a3a73d28b12d9fc..0000000000000000000000000000000000000000
Binary files a/doc/results/old/pr_CHASEDB1.png and /dev/null differ
diff --git a/doc/results/old/pr_DRIVE.png b/doc/results/old/pr_DRIVE.png
deleted file mode 100644
index 2aee69db83b3caaabcaa71e7c24d2eab7ca0eb7e..0000000000000000000000000000000000000000
Binary files a/doc/results/old/pr_DRIVE.png and /dev/null differ
diff --git a/doc/results/old/pr_HRF.png b/doc/results/old/pr_HRF.png
deleted file mode 100644
index df479805f32dcd60770facfab285322d02534c68..0000000000000000000000000000000000000000
Binary files a/doc/results/old/pr_HRF.png and /dev/null differ
diff --git a/doc/results/old/pr_IOSTARVESSEL.png b/doc/results/old/pr_IOSTARVESSEL.png
deleted file mode 100644
index e7c1b9a9203473d50a21adf938c9f4ccb0abe034..0000000000000000000000000000000000000000
Binary files a/doc/results/old/pr_IOSTARVESSEL.png and /dev/null differ
diff --git a/doc/results/old/pr_STARE.png b/doc/results/old/pr_STARE.png
deleted file mode 100644
index c485243f5ecb3350685a327fd61e9a12d457e71d..0000000000000000000000000000000000000000
Binary files a/doc/results/old/pr_STARE.png and /dev/null differ
diff --git a/doc/results/xtest/driu-chasedb1.pdf b/doc/results/xtest/driu-chasedb1.pdf
deleted file mode 100644
index fde498eaa0c30e0b348324953adbbeb621ead281..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-chasedb1.pdf and /dev/null differ
diff --git a/doc/results/xtest/driu-chasedb1.png b/doc/results/xtest/driu-chasedb1.png
deleted file mode 100644
index c921aa43cbe27ff4c576210aab1d27b5857d481b..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-chasedb1.png and /dev/null differ
diff --git a/doc/results/xtest/driu-drive.pdf b/doc/results/xtest/driu-drive.pdf
deleted file mode 100644
index 8b50e32801e9d7d3201daed77f907a7aab8416fc..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-drive.pdf and /dev/null differ
diff --git a/doc/results/xtest/driu-drive.png b/doc/results/xtest/driu-drive.png
deleted file mode 100644
index 091466ee12a0e4b1639e599565b11662807a4442..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-drive.png and /dev/null differ
diff --git a/doc/results/xtest/driu-hrf.pdf b/doc/results/xtest/driu-hrf.pdf
deleted file mode 100644
index efc48a36390dc65cb31b5499dc1b0b82db451065..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-hrf.pdf and /dev/null differ
diff --git a/doc/results/xtest/driu-hrf.png b/doc/results/xtest/driu-hrf.png
deleted file mode 100644
index eebe231b707c126e5e05eba396c1f64b780a6edd..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-hrf.png and /dev/null differ
diff --git a/doc/results/xtest/driu-iostar-vessel.pdf b/doc/results/xtest/driu-iostar-vessel.pdf
deleted file mode 100644
index c4a6fab362dc5dae7cc6462f5c3028241764d7bd..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-iostar-vessel.pdf and /dev/null differ
diff --git a/doc/results/xtest/driu-iostar-vessel.png b/doc/results/xtest/driu-iostar-vessel.png
deleted file mode 100644
index adbfb7352febb35115ee6c558c1a2422015eb721..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-iostar-vessel.png and /dev/null differ
diff --git a/doc/results/xtest/driu-stare.pdf b/doc/results/xtest/driu-stare.pdf
deleted file mode 100644
index 16bdde942b5f3ad1607e5d4fa7b3ff4c77977e8b..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-stare.pdf and /dev/null differ
diff --git a/doc/results/xtest/driu-stare.png b/doc/results/xtest/driu-stare.png
deleted file mode 100644
index de273d697636439d80d00aba7d407f58f44f9e08..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/driu-stare.png and /dev/null differ
diff --git a/doc/results/xtest/index.rst b/doc/results/xtest/index.rst
index 73ebafffe2e3ccead8f89569e918d4ae5ffb127b..b12748512a17dbd3c254516981c5dc24d9112361 100644
--- a/doc/results/xtest/index.rst
+++ b/doc/results/xtest/index.rst
@@ -6,9 +6,6 @@
Cross-Database (X-)Tests
==========================
-F1 Scores (micro-level)
------------------------
-
* Models are trained and tested on the same dataset (numbers in parenthesis
indicate number of parameters per model), and then evaluated across the test
sets of other databases. X-tested datasets therefore represent *unseen*
@@ -25,217 +22,24 @@ F1 Scores (micro-level)
example, to run cross-evaluation tests for the DRIVE dataset, use the
configuration resource :py:mod:`drive-xtest
<bob.ip.binseg.configs.datasets.drive.xtest>`.
-* We only show results for DRIU (~15.4 million parameters) and M2U-Net (~550
- thousand parameters) as these models seem to represent the performance
- extremes according to our :ref:`baseline analysis
+* For each row, the peak performance is always obtained in an intra-database
+ test (training and testing on the same database). Conversely, we observe a
+ performance degradation (albeit not catastrophic in most cases) for all other
+ datasets in the cross test.
+* We only show results for select systems in :ref:`baseline analysis
<bob.ip.binseg.results.baselines>`. You may run analysis on the other models
by downloading them from our website (via the ``--weight`` parameter on the
- :ref:`analyze script <bob.ip.binseg.cli.analyze>`). This script may help you
- in this task, provided you created a directory structure as suggested by
- :ref:`our baseline script <bob.ip.binseg.baseline-script>`:
-
- .. literalinclude:: ../../scripts/xtest.sh
- :language: bash
-
-
-DRIU
-====
-
-
-.. list-table::
- :header-rows: 2
-
- * -
- - drive
- - stare
- - chasedb1
- - hrf
- - iostar-vessel
- * - Model / W x H
- - 544 x 544
- - 704 x 608
- - 960 x 960
- - 1648 x 1168
- - 1024 x 1024
- * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_driu_drive_>`_)
- - **0.819 (0.016)**
- - 0.759 (0.151)
- - 0.321 (0.068)
- - 0.711 (0.067)
- - 0.493 (0.049)
- * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_driu_stare_>`_)
- - 0.733 (0.037)
- - **0.824 (0.037)**
- - 0.491 (0.094)
- - 0.773 (0.051)
- - 0.469 (0.055)
- * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_driu_chase_>`_)
- - 0.730 (0.023)
- - 0.730 (0.101)
- - **0.811 (0.018)**
- - 0.779 (0.043)
- - 0.774 (0.019)
- * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_driu_hrf_>`_)
- - 0.702 (0.038)
- - 0.641 (0.160)
- - 0.600 (0.072)
- - **0.802 (0.039)**
- - 0.546 (0.078)
- * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_driu_iostar_>`_)
- - 0.758 (0.019)
- - 0.724 (0.115)
- - 0.777 (0.032)
- - 0.727 (0.059)
- - **0.825 (0.021)**
-
-
-Next, you will find the PR plots showing confidence intervals, for the various
-cross-tests explored, on a per cross-tested model arrangement. All curves
-correspond to test set performances. Single performance figures (F1-micro
-scores) correspond to its average value across all test set images, for a fixed
-threshold set *a priori* on the training set of dataset used for creating the
-model, and using 100 points for curve calculation.
-
-.. list-table::
-
- * - .. figure:: driu-drive.png
- :align: center
- :scale: 40%
- :alt: X-tests for a DRIU model based on DRIVE
-
- :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.xtest>`: DRIU model X-tested (:download:`pdf <driu-drive.pdf>`)
- - .. figure:: driu-stare.png
- :align: center
- :scale: 40%
- :alt: X-tests for a DRIU model based on STARE
-
- :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.xtest>`: DRIU model X-tested (:download:`pdf <driu-stare.pdf>`)
- * - .. figure:: driu-chasedb1.png
- :align: center
- :scale: 40%
- :alt: X-tests for a DRIU model based on CHASE-DB1
-
- :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.xtest>`: DRIU model X-tested (:download:`pdf <driu-chasedb1.pdf>`)
- - .. figure:: driu-hrf.png
- :align: center
- :scale: 40%
- :alt: X-tests for a DRIU model based on HRF
-
- :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.xtest>`: DRIU model X-tested (:download:`pdf <driu-hrf.pdf>`)
- * - .. figure:: driu-iostar-vessel.png
- :align: center
- :scale: 40%
- :alt: X-tests for a DRIU model based on IOSTAR (vessel)
+ :ref:`analyze script <bob.ip.binseg.cli.analyze>`).
- :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel_xtest>`: DRIU model X-tested (:download:`pdf <driu-iostar-vessel.pdf>`)
- -
+Models on Specific Tasks
+------------------------
-M2U-Net
-=======
+.. toctree::
+ :maxdepth: 2
+ vessel/driu
+ vessel/m2unet
-.. list-table::
- :header-rows: 2
-
- * -
- - drive
- - stare
- - chasedb1
- - hrf
- - iostar-vessel
- * - Model / W x H
- - 544 x 544
- - 704 x 608
- - 960 x 960
- - 1648 x 1168
- - 1024 x 1024
- * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_m2unet_drive_>`_)
- - **0.804 (0.014)**
- - 0.736 (0.144)
- - 0.548 (0.055)
- - 0.744 (0.058)
- - 0.722 (0.036)
- * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_m2unet_stare_>`_)
- - 0.715 (0.031)
- - **0.811 (0.039)**
- - 0.632 (0.033)
- - 0.765 (0.049)
- - 0.673 (0.033)
- * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_m2unet_chase_>`_)
- - 0.677 (0.027)
- - 0.695 (0.099)
- - **0.801 (0.018)**
- - 0.763 (0.040)
- - 0.761 (0.018)
- * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_m2unet_hrf_>`_)
- - 0.591 (0.071)
- - 0.460 (0.230)
- - 0.332 (0.108)
- - **0.796 (0.043)**
- - 0.419 (0.088)
- * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_m2unet_iostar_>`_)
- - 0.743 (0.019)
- - 0.745 (0.076)
- - 0.771 (0.030)
- - 0.749 (0.052)
- - **0.817 (0.021)**
-
-
-Next, you will find the PR plots showing confidence intervals, for the various
-cross-tests explored, on a per cross-tested model arrangement. All curves
-correspond to test set performances. Single performance figures (F1-micro
-scores) correspond to its average value across all test set images, for a fixed
-threshold set *a priori* on the training set of dataset used for creating the
-model.
-
-.. list-table::
-
- * - .. figure:: m2unet-drive.png
- :align: center
- :scale: 40%
- :alt: X-tests for a M2U-Net model based on DRIVE
-
- :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-drive.pdf>`)
- - .. figure:: m2unet-stare.png
- :align: center
- :scale: 40%
- :alt: X-tests for a M2U-Net model based on STARE
-
- :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-stare.pdf>`)
- * - .. figure:: m2unet-chasedb1.png
- :align: center
- :scale: 40%
- :alt: X-tests for a M2U-Net model based on CHASE-DB1
-
- :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-chasedb1.pdf>`)
- - .. figure:: m2unet-hrf.png
- :align: center
- :scale: 40%
- :alt: X-tests for a M2U-Net model based on HRF
-
- :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-hrf.pdf>`)
- * - .. figure:: m2unet-iostar-vessel.png
- :align: center
- :scale: 40%
- :alt: X-tests for a M2U-Net model based on IOSTAR (vessel)
-
- :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel_xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-iostar-vessel.pdf>`)
- -
-
-
-
-Remarks
--------
-
-* For each row, the peak performance is always obtained in an intra-database
- test (training and testing on the same database). Conversely, we observe a
- performance degradation (albeit not catastrophic in most cases) for all other
- datasets in the cross test.
-* X-test performance on a model created from HRF suggests a strong bias, as
- performance does not generalize well for other (unseen) datasets.
-* Models generated from CHASE-DB1 and IOSTAR (vessel) seem to generalize quite
- well to unseen data, when compared to the relatively poor generalization
- capabilites of models generated from HRF or DRIVE.
.. include:: ../../links.rst
diff --git a/doc/results/xtest/m2unet-chasedb1.pdf b/doc/results/xtest/m2unet-chasedb1.pdf
deleted file mode 100644
index efc263a243293184af63d492afa23931b40102b9..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-chasedb1.pdf and /dev/null differ
diff --git a/doc/results/xtest/m2unet-chasedb1.png b/doc/results/xtest/m2unet-chasedb1.png
deleted file mode 100644
index 98edc0b110663ad6d13c138763ddc9656ec737be..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-chasedb1.png and /dev/null differ
diff --git a/doc/results/xtest/m2unet-drive.pdf b/doc/results/xtest/m2unet-drive.pdf
deleted file mode 100644
index c672c163666917083984d1a14836c22184faa3ff..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-drive.pdf and /dev/null differ
diff --git a/doc/results/xtest/m2unet-drive.png b/doc/results/xtest/m2unet-drive.png
deleted file mode 100644
index 68d882506091f5e4cdf50e8f11c6f1720bab8000..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-drive.png and /dev/null differ
diff --git a/doc/results/xtest/m2unet-hrf.pdf b/doc/results/xtest/m2unet-hrf.pdf
deleted file mode 100644
index 226f3af219390f6ce0863df66c1575f79b4cb998..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-hrf.pdf and /dev/null differ
diff --git a/doc/results/xtest/m2unet-hrf.png b/doc/results/xtest/m2unet-hrf.png
deleted file mode 100644
index 19959b3c5aae6e639ae8199343549c4d2f1dc114..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-hrf.png and /dev/null differ
diff --git a/doc/results/xtest/m2unet-iostar-vessel.pdf b/doc/results/xtest/m2unet-iostar-vessel.pdf
deleted file mode 100644
index 4592dc35ba1f668d8b3f5fdb9451b22b2f0dba9b..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-iostar-vessel.pdf and /dev/null differ
diff --git a/doc/results/xtest/m2unet-iostar-vessel.png b/doc/results/xtest/m2unet-iostar-vessel.png
deleted file mode 100644
index 58aa8a5457a482908789b1b04ae82ff249a503e1..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-iostar-vessel.png and /dev/null differ
diff --git a/doc/results/xtest/m2unet-stare.pdf b/doc/results/xtest/m2unet-stare.pdf
deleted file mode 100644
index 13f57fc75575561cebf62ace972749f268530307..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-stare.pdf and /dev/null differ
diff --git a/doc/results/xtest/m2unet-stare.png b/doc/results/xtest/m2unet-stare.png
deleted file mode 100644
index f4ec5207403426c73d003bea7e27b341a2787f3f..0000000000000000000000000000000000000000
Binary files a/doc/results/xtest/m2unet-stare.png and /dev/null differ
diff --git a/doc/results/xtest/vessel/driu.rst b/doc/results/xtest/vessel/driu.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e373366066b5c10fdae8d96e1ae617d78c7a66b4
--- /dev/null
+++ b/doc/results/xtest/vessel/driu.rst
@@ -0,0 +1,57 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.xtest.vessel.driu:
+
+=============================
+ DRIU on Vessel Segmentation
+=============================
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ - drive
+ - stare
+ - chasedb1
+ - hrf
+ - iostar-vessel
+ * - Model / W x H
+ - 544 x 544
+ - 704 x 608
+ - 960 x 960
+ - 1648 x 1168
+ - 1024 x 1024
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_driu_drive_>`_)
+ - **0.819 (0.016)**
+ - 0.759 (0.151)
+ - 0.321 (0.068)
+ - 0.711 (0.067)
+ - 0.493 (0.049)
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_driu_stare_>`_)
+ - 0.733 (0.037)
+ - **0.824 (0.037)**
+ - 0.491 (0.094)
+ - 0.773 (0.051)
+ - 0.469 (0.055)
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_driu_chase_>`_)
+ - 0.730 (0.023)
+ - 0.730 (0.101)
+ - **0.811 (0.018)**
+ - 0.779 (0.043)
+ - 0.774 (0.019)
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_driu_hrf_>`_)
+ - 0.702 (0.038)
+ - 0.641 (0.160)
+ - 0.600 (0.072)
+ - **0.802 (0.039)**
+ - 0.546 (0.078)
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_driu_iostar_>`_)
+ - 0.758 (0.019)
+ - 0.724 (0.115)
+ - 0.777 (0.032)
+ - 0.727 (0.059)
+ - **0.825 (0.021)**
+
+
+.. include:: ../../../links.rst
diff --git a/doc/results/xtest/vessel/m2unet.rst b/doc/results/xtest/vessel/m2unet.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c253ad9447646306990bf0e9633ad0f55b73e7f0
--- /dev/null
+++ b/doc/results/xtest/vessel/m2unet.rst
@@ -0,0 +1,57 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.xtest.vessel.m2unet:
+
+================================
+ M2U-Net on Vessel Segmentation
+================================
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ - drive
+ - stare
+ - chasedb1
+ - hrf
+ - iostar-vessel
+ * - Model / W x H
+ - 544 x 544
+ - 704 x 608
+ - 960 x 960
+ - 1648 x 1168
+ - 1024 x 1024
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_m2unet_drive_>`_)
+ - **0.804 (0.014)**
+ - 0.736 (0.144)
+ - 0.548 (0.055)
+ - 0.744 (0.058)
+ - 0.722 (0.036)
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_m2unet_stare_>`_)
+ - 0.715 (0.031)
+ - **0.811 (0.039)**
+ - 0.632 (0.033)
+ - 0.765 (0.049)
+ - 0.673 (0.033)
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_m2unet_chase_>`_)
+ - 0.677 (0.027)
+ - 0.695 (0.099)
+ - **0.801 (0.018)**
+ - 0.763 (0.040)
+ - 0.761 (0.018)
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_m2unet_hrf_>`_)
+ - 0.591 (0.071)
+ - 0.460 (0.230)
+ - 0.332 (0.108)
+ - **0.796 (0.043)**
+ - 0.419 (0.088)
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_m2unet_iostar_>`_)
+ - 0.743 (0.019)
+ - 0.745 (0.076)
+ - 0.771 (0.030)
+ - 0.749 (0.052)
+ - **0.817 (0.021)**
+
+
+.. include:: ../../../links.rst
diff --git a/doc/scripts/baselines.sh b/doc/scripts/baselines.sh
deleted file mode 100755
index 20cc36f30023193e8c46362de04f3419b9b735cc..0000000000000000000000000000000000000000
--- a/doc/scripts/baselines.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-# Runs all of our baselines
-
-# set output directory and location of "bob" executable
-OUTDIR=/path/where/to/dump/results
-BOB=/path/to/bob
-
-# run <modelconfig> <dbconfig> <batchsize> [<device> [<queue>]]
-function run() {
- local device="cpu"
- [ $# -gt 3 ] && device="${4}"
-
- local cmd=(${BOB} binseg experiment)
- cmd+=("-vv" "--device=${device}" ${1} ${2})
- cmd+=("--batch-size=${3}" "--output-folder=${OUTDIR}/${1}/${2}")
-
- mkdir -pv ${OUTDIR}/${1}/${2}
-
- [ $# -gt 4 ] && cmd=(jman submit "--log-dir=${OUTDIR}/${1}/${2}" "--name=$(basename ${OUTDIR})-${1}-${2}" "--memory=24G" "--queue=${5}" -- "${cmd[@]}")
-
- if [ $# -le 4 ]; then
- # executing locally, capture stdout and stderr
- ("${cmd[@]}" | tee "${OUTDIR}/${1}/${2}/stdout.log") 3>&1 1>&2 2>&3 | tee "${OUTDIR}/${1}/${2}/stderr.log"
- else
- "${cmd[@]}"
- fi
-}
-
-
-# run/submit all baselines
-# comment out from "cuda:0" to run on CPU
-# comment out from "sgpu/gpu" to run locally
-run m2unet drive 16 #cuda:0 #sgpu
-run hed drive 8 #cuda:0 #sgpu
-run driu drive 8 #cuda:0 #sgpu
-run unet drive 4 #cuda:0 #sgpu
-run lwnet drive 4 #cuda:0 #sgpu
-run m2unet stare 6 #cuda:0 #sgpu
-run hed stare 4 #cuda:0 #sgpu
-run driu stare 5 #cuda:0 #sgpu
-run unet stare 2 #cuda:0 #sgpu
-run lwnet stare 4 #cuda:0 #sgpu
-run m2unet chasedb1 6 #cuda:0 #sgpu
-run hed chasedb1 4 #cuda:0 #sgpu
-run driu chasedb1 4 #cuda:0 #sgpu
-run unet chasedb1 2 #cuda:0 #sgpu
-run lwnet chasedb1 4 #cuda:0 #sgpu
-run m2unet hrf 1 #cuda:0 # gpu
-run hed hrf 1 #cuda:0 # gpu
-run driu hrf 1 #cuda:0 # gpu
-run unet hrf 1 #cuda:0 # gpu
-run lwnet hrf 4 #cuda:0 # gpu
-run m2unet iostar-vessel 6 #cuda:0 # gpu
-run hed iostar-vessel 4 #cuda:0 # gpu
-run driu iostar-vessel 4 #cuda:0 # gpu
-run unet iostar-vessel 2 #cuda:0 # gpu
-run lwnet iostar-vessel 4 #cuda:0 # gpu
diff --git a/doc/scripts/functions.sh b/doc/scripts/functions.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ead6341a1c26810892f93a4fd4df31a0b5cea495
--- /dev/null
+++ b/doc/scripts/functions.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# set output directory and location of "bob" executable
+OUTDIR=/path/where/to/dump/results
+BOB=/path/to/bob
+
+# this function just makes running/submitting jobs a bit easier for extensive
+# benchmark jobs.
+# usage: run <modelconfig> <dbconfig> <batchsize> [<device> [<queue>]]
+function run() {
+ local device="cpu"
+ [ $# -gt 3 ] && device="${4}"
+
+ local cmd=(${BOB} binseg experiment)
+ cmd+=("-vv" "--device=${device}" ${1} ${2})
+ cmd+=("--batch-size=${3}" "--output-folder=${OUTDIR}/${1}/${2}")
+ # add --multiproc-data-loading=0 to increase data loading/transform speeds,
+ # but pay by making your results harder to reproduce (OS-random data loading)
+ #cmd+=("--multiproc-data-loading=0")
+
+ mkdir -pv ${OUTDIR}/${1}/${2}
+
+ [ $# -gt 4 ] && cmd=(jman submit "--log-dir=${OUTDIR}/${1}/${2}" "--name=$(basename ${OUTDIR})-${1}-${2}" "--memory=24G" "--queue=${5}" -- "${cmd[@]}")
+
+ if [ $# -le 4 ]; then
+ # executing locally, capture stdout and stderr
+ ("${cmd[@]}" | tee "${OUTDIR}/${1}/${2}/stdout.log") 3>&1 1>&2 2>&3 | tee "${OUTDIR}/${1}/${2}/stderr.log"
+ else
+ "${cmd[@]}"
+ fi
+}
diff --git a/setup.py b/setup.py
index 33cb7e73ee4cf811ffe3b5a4c0731f150332ce5f..58c0a1c8889bd374587f46d98bab9e6280058c64 100644
--- a/setup.py
+++ b/setup.py
@@ -57,59 +57,69 @@ setup(
"lwnet = bob.ip.binseg.configs.models.lwnet",
# example datasets
"csv-dataset-example = bob.ip.binseg.configs.datasets.csv",
- # drive dataset
+ # drive dataset - retinography
"drive = bob.ip.binseg.configs.datasets.drive.default",
"drive-2nd = bob.ip.binseg.configs.datasets.drive.second_annotator",
"drive-xtest = bob.ip.binseg.configs.datasets.drive.xtest",
"drive-mtest = bob.ip.binseg.configs.datasets.drive.mtest",
"drive-covd = bob.ip.binseg.configs.datasets.drive.covd",
"drive-ssl = bob.ip.binseg.configs.datasets.drive.ssl",
- # drhagis dataset
+ # drhagis dataset - retinography
"drhagis = bob.ip.binseg.configs.datasets.drhagis.default",
- # stare dataset
+ # stare dataset - retinography
"stare = bob.ip.binseg.configs.datasets.stare.ah",
"stare-2nd = bob.ip.binseg.configs.datasets.stare.vk",
"stare-xtest = bob.ip.binseg.configs.datasets.stare.xtest",
"stare-mtest = bob.ip.binseg.configs.datasets.stare.mtest",
"stare-covd = bob.ip.binseg.configs.datasets.stare.covd",
"stare-ssl = bob.ip.binseg.configs.datasets.stare.ssl",
- # iostar
+ # iostar - retinography
"iostar-vessel = bob.ip.binseg.configs.datasets.iostar.vessel",
"iostar-vessel-xtest = bob.ip.binseg.configs.datasets.iostar.vessel_xtest",
"iostar-vessel-mtest = bob.ip.binseg.configs.datasets.iostar.vessel_mtest",
"iostar-disc = bob.ip.binseg.configs.datasets.iostar.optic_disc",
"iostar-vessel-covd = bob.ip.binseg.configs.datasets.iostar.covd",
"iostar-vessel-ssl = bob.ip.binseg.configs.datasets.iostar.ssl",
- # hrf
+ # hrf - retinography
"hrf = bob.ip.binseg.configs.datasets.hrf.default",
"hrf-xtest = bob.ip.binseg.configs.datasets.hrf.xtest",
"hrf-mtest = bob.ip.binseg.configs.datasets.hrf.mtest",
"hrf-highres = bob.ip.binseg.configs.datasets.hrf.default_fullres",
"hrf-covd = bob.ip.binseg.configs.datasets.hrf.covd",
"hrf-ssl = bob.ip.binseg.configs.datasets.hrf.ssl",
- # chase-db1
+ # chase-db1 - retinography
"chasedb1 = bob.ip.binseg.configs.datasets.chasedb1.first_annotator",
"chasedb1-2nd = bob.ip.binseg.configs.datasets.chasedb1.second_annotator",
"chasedb1-xtest = bob.ip.binseg.configs.datasets.chasedb1.xtest",
"chasedb1-mtest = bob.ip.binseg.configs.datasets.chasedb1.mtest",
"chasedb1-covd = bob.ip.binseg.configs.datasets.chasedb1.covd",
"chasedb1-ssl = bob.ip.binseg.configs.datasets.chasedb1.ssl",
- # drionsdb
+ # drionsdb - retinography
"drionsdb = bob.ip.binseg.configs.datasets.drionsdb.expert1",
"drionsdb-2nd = bob.ip.binseg.configs.datasets.drionsdb.expert2",
- # drishti-gs1
+ # drishti-gs1 - retinography
"drishtigs1-disc = bob.ip.binseg.configs.datasets.drishtigs1.disc_all",
"drishtigs1-cup = bob.ip.binseg.configs.datasets.drishtigs1.cup_all",
"drishtigs1-disc-any = bob.ip.binseg.configs.datasets.drishtigs1.disc_any",
"drishtigs1-cup-any = bob.ip.binseg.configs.datasets.drishtigs1.cup_any",
- # refuge
+ # refuge - retinography
"refuge-cup = bob.ip.binseg.configs.datasets.refuge.cup",
"refuge-disc = bob.ip.binseg.configs.datasets.refuge.disc",
- # rim one r3
+ # rim one r3 - retinography
"rimoner3-cup = bob.ip.binseg.configs.datasets.rimoner3.cup_exp1",
"rimoner3-disc = bob.ip.binseg.configs.datasets.rimoner3.disc_exp1",
"rimoner3-cup-2nd = bob.ip.binseg.configs.datasets.rimoner3.cup_exp2",
"rimoner3-disc-2nd = bob.ip.binseg.configs.datasets.rimoner3.disc_exp2",
+ # montgomery county - cxr
+ "montgomery = bob.ip.binseg.configs.datasets.montgomery.default",
+ "montgomery-xtest = bob.ip.binseg.configs.datasets.montgomery.xtest",
+ # shenzhen - cxr
+ "shenzhen = bob.ip.binseg.configs.datasets.shenzhen.default",
+ "shenzhen-small = bob.ip.binseg.configs.datasets.shenzhen.default_256",
+ "shenzhen-xtest = bob.ip.binseg.configs.datasets.shenzhen.xtest",
+ # jsrt - cxr
+ "jsrt = bob.ip.binseg.configs.datasets.jsrt.default",
+ "jsrt-xtest = bob.ip.binseg.configs.datasets.jsrt.xtest",
],
},
# check classifiers, add and remove as you see fit