diff --git a/.gitignore b/.gitignore
index fcc2be3d0eb5cbef10d057c51da065a6f947ddfd..fbdcee4cc0c8d8c8736864b4d8065d5d3aec2467 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-### Bob defaults ###
*~
*.swp
*.pyc
@@ -18,26 +17,15 @@ dist
build
*.egg
src/
+doc/api
record.txt
core
output_temp
output
*.DS_Store
-
-
-### JupyterNotebook ###
*.ipynb
.ipynb_checkpoints
*/.ipynb_checkpoints/*
-
-
-### VisualStudioCode ###
-.vscode/*
-.vscode/settings.json
-.vscode/tasks.json
-.vscode/launch.json
-.vscode/extensions.json
-
-### VisualStudioCode Patch ###
-# Ignore all local history of files
-.history
\ No newline at end of file
+submitted.sql3
+./logs/
+.coverage
diff --git a/MANIFEST.in b/MANIFEST.in
index 946a661e098c89b0b5b505c5b22594f474fd8b9a..70f6ad2ea295cdf025eab2310e578179315433f1 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,3 @@
include README.rst buildout.cfg COPYING version.txt requirements.txt
-recursive-include doc *.rst *.png *.ico *.txt
+recursive-include doc *.sh *.rst *.png *.pdf *.ico *.txt
+recursive-include bob *.json *.png *.csv *.jpg
diff --git a/README.rst b/README.rst
index 9ce05003a7e41e5e0b9b9e8116745884305e1da6..bb0badec1ef9f6f3fb29e033c37ca0288c9f4afb 100644
--- a/README.rst
+++ b/README.rst
@@ -4,10 +4,10 @@
:target: https://www.idiap.ch/software/bob/docs/bob/bob.ip.binseg/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/bob/docs/bob/bob.ip.binseg/master/index.html
-.. image:: https://gitlab.idiap.ch/bob/bob.ip.binseg/badges/master/build.svg
+.. image:: https://gitlab.idiap.ch/bob/bob.ip.binseg/badges/master/pipeline.svg
:target: https://gitlab.idiap.ch/bob/bob.ip.binseg/commits/master
.. image:: https://gitlab.idiap.ch/bob/bob.ip.binseg/badges/master/coverage.svg
- :target: https://gitlab.idiap.ch/bob/bob.ip.binseg/commits/master
+ :target: https://www.idiap.ch/software/bob/docs/bob/bob.ip.binseg/master/coverage/index.html
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.ip.binseg
.. image:: https://img.shields.io/pypi/v/bob.ip.binseg.svg
@@ -38,7 +38,7 @@ If you use this software package in a publication, we would appreciate if you
could cite our work::
@misc{laibacher_anjos_2019,
- title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Funduscopy},
+ title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Retinography},
author = {Tim Laibacher and Andr\'e Anjos},
year = {2019},
eprint = {1909.03856},
diff --git a/bob/__init__.py b/bob/__init__.py
index 2ab1e28b150f0549def9963e9e87de3fdd6b2579..edbb4090fca046b19d22d3982711084621bff3be 100644
--- a/bob/__init__.py
+++ b/bob/__init__.py
@@ -1,3 +1,4 @@
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
+
__path__ = extend_path(__path__, __name__)
diff --git a/bob/ip/__init__.py b/bob/ip/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..edbb4090fca046b19d22d3982711084621bff3be 100644
--- a/bob/ip/__init__.py
+++ b/bob/ip/__init__.py
@@ -1,3 +1,4 @@
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
+
+__path__ = extend_path(__path__, __name__)
diff --git a/bob/ip/binseg/__init__.py b/bob/ip/binseg/__init__.py
index 8e48dba233d1b4e018d5dc08d3eb18aacc86138e..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/__init__.py
+++ b/bob/ip/binseg/__init__.py
@@ -1,17 +0,0 @@
-# gets sphinx autodoc done right - don't remove it
-def __appropriate__(*args):
- """Says object was actually declared here, an not on the import module.
-
- Parameters:
-
- *args: An iterable of objects to modify
-
- Resolves `Sphinx referencing issues
- <https://github.com/sphinx-doc/sphinx/issues/3048>`
- """
-
- for obj in args: obj.__module__ = __name__
-
-__appropriate__()
-
-__all__ = [_ for _ in dir() if not _.startswith('_')]
diff --git a/bob/ip/binseg/configs/__init__.py b/bob/ip/binseg/configs/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/configs/__init__.py
+++ b/bob/ip/binseg/configs/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/__init__.py b/bob/ip/binseg/configs/datasets/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..cd6d9b34db1ac30aff072a947465a016d413b37f 100644
--- a/bob/ip/binseg/configs/datasets/__init__.py
+++ b/bob/ip/binseg/configs/datasets/__init__.py
@@ -1,3 +1,176 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Standard configurations for dataset setup"""
+
+
+from ...data.transforms import (
+ RandomRotation as _rotation,
+ RandomHorizontalFlip as _hflip,
+ RandomVerticalFlip as _vflip,
+ ColorJitter as _jitter,
+)
+
+
+RANDOM_ROTATION = [_rotation()]
+"""Shared data augmentation based on random rotation only"""
+
+
+RANDOM_FLIP_JITTER = [_hflip(), _vflip(), _jitter()]
+"""Shared data augmentation transforms without random rotation"""
+
+
+def make_subset(l, transforms, prefixes=[], suffixes=[]):
+ """Creates a new data set, applying transforms
+
+ .. note::
+
+ This is a convenience function for our own dataset definitions inside
+ this module, guaranteeting homogenity between dataset definitions
+ provided in this package. It assumes certain strategies for data
+ augmentation that may not be translatable to other applications.
+
+
+ Parameters
+ ----------
+
+ l : list
+ List of delayed samples
+
+ transforms : list
+ A list of transforms that needs to be applied to all samples in the set
+
+ prefixes : list
+ A list of data augmentation operations that needs to be applied
+ **before** the transforms above
+
+ suffixes : list
+ A list of data augmentation operations that needs to be applied
+ **after** the transforms above
+
+
+ Returns
+ -------
+
+ subset : :py:class:`bob.ip.binseg.data.utils.SampleListDataset`
+ A pre-formatted dataset that can be fed to one of our engines
+
+ """
+
+ from ...data.utils import SampleListDataset as wrapper
+
+ return wrapper(l, prefixes + transforms + suffixes)
+
+
+def make_trainset(l, transforms, rotation_before=False):
+ """Creates a new training set, **with data augmentation**
+
+ Typically, the transforms are chained to a default set of data augmentation
+ operations (random rotation, horizontal and vertical flips, and color
+ jitter), but flag allows prefixing the rotation specially (useful for some
+ COVD training sets).
+
+ .. note::
+
+ This is a convenience function for our own dataset definitions inside
+ this module, guaranteeting homogenity between dataset definitions
+ provided in this package. It assumes certain strategies for data
+ augmentation that may not be translatable to other applications.
+
+
+ Parameters
+ ----------
+
+ l : list
+ List of delayed samples
+
+ transforms : list
+ A list of transforms that needs to be applied to all samples in the set
+
+
+ Returns
+ -------
+
+ subset : :py:class:`bob.ip.binseg.data.utils.SampleListDataset`
+ A pre-formatted dataset that can be fed to one of our engines
+
+ """
+
+ if rotation_before:
+ return make_subset(
+ l,
+ transforms=transforms,
+ prefixes=RANDOM_ROTATION,
+ suffixes=RANDOM_FLIP_JITTER,
+ )
+
+ return make_subset(
+ l,
+ transforms=transforms,
+ suffixes=(RANDOM_ROTATION + RANDOM_FLIP_JITTER),
+ )
+
+
+def make_dataset(subsets, transforms):
+ """Creates a new configuration dataset from dictionary and transforms
+
+ This function takes as input a dictionary as those that can be returned by
+ :py:meth:`bob.ip.binseg.data.dataset.JSONDataset.subsets`, or
+ :py:meth:`bob.ip.binseg.data.dataset.CSVDataset.subsets`, mapping protocol
+ names (such as ``train``, ``dev`` and ``test``) to
+ :py:class:`bob.ip.binseg.data.sample.DelayedSample` lists, and a set of
+ transforms, and returns a dictionary applying
+ :py:class:`bob.ip.binseg.data.utils.SampleListDataset` to these
+ lists, and our standard data augmentation if a ``train`` set exists.
+
+ For example, if ``subsets`` is composed of two sets named ``train`` and
+ ``test``, this function will yield a dictionary with the following entries:
+
+ * ``__train__``: Wraps the ``train`` subset, includes data augmentation
+ (note: datasets with names starting with ``_`` (underscore) are excluded
+ from prediction and evaluation by default, as they contain data
+ augmentation transformations.)
+ * ``train``: Wraps the ``train`` subset, **without** data augmentation
+ * ``train``: Wraps the ``test`` subset, **without** data augmentation
+
+ .. note::
+
+ This is a convenience function for our own dataset definitions inside
+ this module, guaranteeting homogenity between dataset definitions
+ provided in this package. It assumes certain strategies for data
+ augmentation that may not be translatable to other applications.
+
+
+ Parameters
+ ----------
+
+ subsets : dict
+ A dictionary that contains the delayed sample lists for a number of
+ named lists. If one of the keys is ``train``, our standard dataset
+ augmentation transforms are appended to the definition of that subset.
+ All other subsets remain un-augmented.
+
+ transforms : list
+ A list of transforms that needs to be applied to all samples in the set
+
+
+ Returns
+ -------
+
+ dataset : dict
+ A pre-formatted dataset that can be fed to one of our engines. It maps
+ string names to
+ :py:class:`bob.ip.binseg.data.utils.SampleListDataset`'s.
+
+ """
+
+ retval = {}
+
+ for key in subsets.keys():
+ retval[key] = make_subset(subsets[key], transforms=transforms)
+ if key == "train":
+ retval["__train__"] = make_trainset(
+ subsets[key], transforms=transforms, rotation_before=False
+ )
+
+ return retval
diff --git a/bob/ip/binseg/configs/datasets/amdrive.py b/bob/ip/binseg/configs/datasets/amdrive.py
deleted file mode 100644
index 6f7cc3101d158f71c7ea47f32382ce369de0552b..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/amdrive.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from bob.db.drive import Database as DRIVE
-from bob.db.stare import Database as STARE
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.db.iostar import Database as IOSTAR
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-import torch
-
-# Target size: 544x544 (DRIVE)
-
-defaulttransforms = [RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()]
-
-
-
-# CHASE_DB1
-transforms_chase = Compose([
- Resize(544)
- ,Crop(0,12,544,544)
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_chase = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-torch_chase = BinSegDataset(bobdb_chase, split='train', transform=transforms_chase)
-
-
-# IOSTAR VESSEL
-transforms_iostar = Compose([
- Resize(544)
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_iostar = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-torch_iostar = BinSegDataset(bobdb_iostar, split='train', transform=transforms_iostar)
-
-# STARE
-transforms = Compose([
- Resize(471)
- ,Pad((0,37,0,36))
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_stare = STARE(protocol = 'default')
-
-# PyTorch dataset
-torch_stare = BinSegDataset(bobdb_stare, split='train', transform=transforms)
-
-
-# HRF
-transforms_hrf = Compose([
- Resize((363))
- ,Pad((0,90,0,91))
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_hrf = HRF(protocol = 'default')
-
-# PyTorch dataset
-torch_hrf = BinSegDataset(bobdb_hrf, split='train', transform=transforms_hrf)
-
-
-
-# Merge
-dataset = torch.utils.data.ConcatDataset([torch_stare, torch_chase, torch_iostar, torch_hrf])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/amdrivetest.py b/bob/ip/binseg/configs/datasets/amdrivetest.py
deleted file mode 100644
index 026ac236e7e9f48f5f41b6685d01369606c47321..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/amdrivetest.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from bob.db.drive import Database as DRIVE
-from bob.db.stare import Database as STARE
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.db.iostar import Database as IOSTAR
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-import torch
-
-# Target size: 544x544 (DRIVE)
-
-defaulttransforms = [ToTensor()]
-
-
-# CHASE_DB1
-transforms_chase = Compose([
- Resize(544)
- ,Crop(0,12,544,544)
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_chase = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-torch_chase = BinSegDataset(bobdb_chase, split='test', transform=transforms_chase)
-
-
-# IOSTAR VESSEL
-transforms_iostar = Compose([
- Resize(544)
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_iostar = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-torch_iostar = BinSegDataset(bobdb_iostar, split='test', transform=transforms_iostar)
-
-# STARE
-transforms = Compose([
- Resize(471)
- ,Pad((0,37,0,36))
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_stare = STARE(protocol = 'default')
-
-# PyTorch dataset
-torch_stare = BinSegDataset(bobdb_stare, split='test', transform=transforms)
-
-
-# HRF
-transforms_hrf = Compose([
- Resize((363))
- ,Pad((0,90,0,91))
- ,*defaulttransforms
- ])
-
-# bob.db.dataset init
-bobdb_hrf = HRF(protocol = 'default')
-
-# PyTorch dataset
-torch_hrf = BinSegDataset(bobdb_hrf, split='test', transform=transforms_hrf)
-
-
-
-# Merge
-dataset = torch.utils.data.ConcatDataset([torch_stare, torch_chase, torch_iostar, torch_hrf])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb1.py b/bob/ip/binseg/configs/datasets/chasedb1.py
deleted file mode 100644
index 7fa0dc096aa55a8ab19665af45c9f2e1f8368a0c..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb1.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,18,960,960)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/__init__.py b/bob/ip/binseg/configs/datasets/chasedb1/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..00e2c80e7a5d79bd2bda1fa0c1c21097bf11ed3a
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/__init__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import Crop
+ from ....data.chasedb1 import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Crop(0, 18, 960, 960)])
+
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/covd.py b/bob/ip/binseg/configs/datasets/chasedb1/covd.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed8c37aff14036c88fc5949820efb733e802e4ae
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/covd.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-CHASEDB1 for Vessel Segmentation
+
+* Configuration resolution (height x width): 960 x 960
+
+The dataset available in this file is composed of DRIVE, STARE, IOSTAR
+vessel and HRF (with annotated samples).
+
+For details on those datasets, consult:
+
+* See :py:mod:`bob.ip.binseg.data.drive`
+* See :py:mod:`bob.ip.binseg.data.stare`
+* See :py:mod:`bob.ip.binseg.data.iostar`
+* See :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Pad, Resize
+from bob.ip.binseg.configs.datasets import make_trainset as _maker
+
+from bob.ip.binseg.data.drive import dataset as _raw_drive
+
+_drive = _maker(
+ _raw_drive.subsets("default")["train"],
+ [CenterCrop((544, 544)), Resize(960)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.stare import dataset as _raw_stare
+
+# n.b.: not the best fit, but what was there for Tim's work
+_stare = _maker(
+ _raw_stare.subsets("ah")["train"],
+ [Pad((0, 32, 0, 32)), Resize(960), CenterCrop(960)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.hrf import dataset as _raw_hrf
+
+_hrf = _maker(
+ _raw_hrf.subsets("default")["train"], [Pad((0, 584, 0, 584)), Resize(960)],
+)
+
+from bob.ip.binseg.data.iostar import dataset as _raw_iostar
+
+# n.b.: not the best fit, but what was there for Tim's work
+_iostar = _maker(_raw_iostar.subsets("vessel")["train"], [Resize(960)])
+
+from torch.utils.data import ConcatDataset
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _baseline,
+)
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_baseline)
+dataset["__train__"] = ConcatDataset([_drive, _stare, _hrf, _iostar])
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/first_annotator.py b/bob/ip/binseg/configs/datasets/chasedb1/first_annotator.py
new file mode 100644
index 0000000000000000000000000000000000000000..07c5684212db6a510a32b246c73a0d7bbe0f8c70
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/first_annotator.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""CHASE-DB1 dataset for Vessel Segmentation (first-annotator protocol)
+
+* Split reference: [CHASEDB1-2012]_
+* Configuration resolution: 960 x 960 (after hand-specified crop)
+* See :py:mod:`bob.ip.binseg.data.chasedb1` for dataset details
+* This dataset offers a second-annotator comparison
+"""
+
+from bob.ip.binseg.configs.datasets.chasedb1 import _maker
+dataset = _maker("first-annotator")
+second_annotator = _maker("second-annotator")
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/mtest.py b/bob/ip/binseg/configs/datasets/chasedb1/mtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..0860d97209d2dcc4ceef736d9f962ddd5dea4776
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/mtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""CHASE-DB1 cross-evaluation dataset with matched resolution
+
+* Configuration resolution (height x width): 960 x 960
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Pad, Resize
+from bob.ip.binseg.configs.datasets.chasedb1.xtest import dataset as _xt
+
+dataset = {
+ "train": _xt["train"],
+ "test": _xt["test"],
+ "drive": _xt["drive"].copy([CenterCrop((544, 544)), Resize(960)]),
+ "stare": _xt["stare"].copy(
+ [Pad((0, 32, 0, 32)), Resize(960), CenterCrop(960)]
+ ),
+ "hrf": _xt["hrf"].copy([Pad((0, 584, 0, 584)), Resize(960)]),
+ "iostar": _xt["iostar"].copy([Resize(960)]),
+}
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/second_annotator.py b/bob/ip/binseg/configs/datasets/chasedb1/second_annotator.py
new file mode 100644
index 0000000000000000000000000000000000000000..87a98310bbbe2c6784a0d4d525d58a6126543e2c
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/second_annotator.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""CHASE-DB1 dataset for Vessel Segmentation (second-annotator protocol)
+
+* Split reference: [CHASEDB1-2012]_
+* Configuration resolution: 960 x 960 (after hand-specified crop)
+* See :py:mod:`bob.ip.binseg.data.chasedb1` for dataset details
+* This dataset offers a second-annotator comparison (using "first-annotator")
+"""
+
+from bob.ip.binseg.configs.datasets.chasedb1 import _maker
+dataset = _maker("second-annotator")
+second_annotator = _maker("first-annotator")
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/ssl.py b/bob/ip/binseg/configs/datasets/chasedb1/ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..8bd97ddad2e8a468ec78d20c99b79240deec4f11
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/ssl.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-CHASE-DB1 + SSL for Vessel Segmentation
+
+* Configuration resolution: 960 x 960
+
+The dataset available in this file is composed of DRIVE, STARE, IOSTAR vessel
+and HRF (with annotated samples) and CHASE-DB1's "first-annotator" training set
+without labels, for training, and CHASE-DB1's "first-annotator" test set, for
+evaluation.
+
+For details on datasets, consult:
+
+* :py:mod:`bob.ip.binseg.data.stare`
+* :py:mod:`bob.ip.binseg.data.drive`
+* :py:mod:`bob.ip.binseg.data.chasedb1`
+* :py:mod:`bob.ip.binseg.data.iostar`
+* :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.configs.datasets.chasedb1.covd import dataset as _covd
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _baseline,
+)
+from bob.ip.binseg.data.utils import SSLDataset
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_covd)
+dataset["__train__"] = SSLDataset(_covd["__train__"], _baseline["__train__"])
diff --git a/bob/ip/binseg/configs/datasets/chasedb1/xtest.py b/bob/ip/binseg/configs/datasets/chasedb1/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..cade7b85c1a4f600fd1edcc8558e326c6a4c8d10
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/chasedb1/xtest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""CHASE-DB1 cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _drive
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _stare
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _chase,
+)
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _hrf
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _iostar
+
+dataset = {
+ "train": _chase["train"],
+ "test": _chase["test"],
+ "drive": _drive["test"],
+ "stare": _stare["test"],
+ "hrf": _hrf["test"],
+ "iostar": _iostar["test"],
+ }
diff --git a/bob/ip/binseg/configs/datasets/chasedb11024.py b/bob/ip/binseg/configs/datasets/chasedb11024.py
deleted file mode 100644
index 028f10fb443cd4f1c9c728203f13641cc67b95ad..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb11024.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Crop(0,18,960,960)
- ,Resize(1024)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb11168.py b/bob/ip/binseg/configs/datasets/chasedb11168.py
deleted file mode 100644
index d221ea4879c4e7378e0f9ad6bcce2cd5a77bf04c..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb11168.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Crop(140,18,680,960)
- ,Resize(1168)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb1544.py b/bob/ip/binseg/configs/datasets/chasedb1544.py
deleted file mode 100644
index 9632d53996d343deecd0e021fb2140153df9edc0..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb1544.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize(544)
- ,Crop(0,12,544,544)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb1608.py b/bob/ip/binseg/configs/datasets/chasedb1608.py
deleted file mode 100644
index 9a475cae5d7fb5be49a28ae6e479dd3fe41760ed..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb1608.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,CenterCrop((829,960))
- ,Resize(608)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/chasedb1test.py b/bob/ip/binseg/configs/datasets/chasedb1test.py
deleted file mode 100644
index 4b267b0f022030d512197f89401f74e3373df3cb..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/chasedb1test.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.chasedb1 import Database as CHASEDB1
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,18,960,960)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = CHASEDB1(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/csv.py b/bob/ip/binseg/configs/datasets/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b62ec89e4f373d811e674a5e471f79e24cd9ced
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/csv.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Example CSV-based custom filelist dataset
+
+In case you have your own dataset that is organized on your filesystem (or
+elsewhere), this configuration shows an example setup so you can feed such data
+(potentially including any ground-truth you may have) to train, predict or
+evaluate one of the available network models.
+
+You must write CSV based file (e.g. using comma as separator) that describes
+the data (and ground-truth) locations for each sample on your dataset. So, for
+example, if you have a file structure like this:
+
+.. code-block:: text
+
+ ├── images
+ ├── image_1.png
+ ├── ...
+ └── image_n.png
+ └── ground-truth
+ ├── gt_1.png
+ ├── ...
+ └── gt_n.png
+
+Then create one or more files, each containing a subset of your dataset:
+
+.. code-block:: text
+
+ images/image_1.png,ground-truth/gt_1.png
+ ...,...
+ images/image_n.png,ground-truth/gt_n.png
+
+To create a subset without ground-truth (e.g., for prediction purposes), then
+omit the second column on the CSV file.
+
+Use the path leading to the CSV file and carefully read the comments in this
+configuration. **Copy it locally to make changes**:
+
+.. code-block:: sh
+
+ $ bob binseg config copy csv-dataset-example mydataset.py
+ # edit mydataset.py as explained here, follow the comments
+
+Finally, the only object this file needs to provide is one named ``dataset``,
+and it should contain a dictionary mapping a name, such as ``train``, ``dev``,
+or ``test``, to objects of type :py:class:`torch.utils.data.Dataset`. As you
+will see in this example, we provide boilerplate code to do so.
+
+More information:
+
+* :py:class:`bob.ip.binseg.data.dataset.CSVDataset` for operational details.
+* :py:class:`bob.ip.binseg.data.dataset.JSONDataset` for an alternative for
+ multi-protocol datasets (all of our supported raw datasets are implemented
+ using this)
+* :py:func:`bob.ip.binseg.configs.datasets.make_dataset` for extra
+ information on the sample list to pytorch connector.
+
+"""
+
+import os
+
+# First, define how to access and load the raw data. Our package provides some
+# stock loaders we use for other datasets. You may have a look at the
+# documentation of that module for details.
+from bob.ip.binseg.data.loader import (
+ load_pil_rgb,
+ load_pil_1,
+)
+
+from bob.ip.binseg.data.sample import Sample
+
+# How we use the loaders - "sample" is a dictionary where keys are defined
+# below and map to the columns of the CSV files you input. This one is
+# configured to load images and labels using PIL.
+def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # it is a dictionary that passes e.g., the name of the subset
+ # being loaded, so you can take contextual decisions on the loading
+
+ # Using the path leading to the various data files stored in disk allows
+ # the CSV file to contain only relative paths and is, therefore, more
+ # compact. Of course, you can make those paths absolute and then simplify
+ # it here.
+ root_path = "/path/where/raw/files/sit"
+
+ data=load_pil_rgb(os.path.join(root_path, sample["data"]))
+ label=load_pil_1(os.path.join(root_path, sample["label"]))
+
+ # You may also return DelayedSample to avoid data loading to take place
+ # as the sample object itself is created. Take a look at our own datasets
+ # for examples.
+ return Sample(
+ key=os.path.splitext(sample["data"])[0],
+ data=dict(data=data, label=label),
+ )
+
+
+# This is just a class that puts everything together: the CSV file, how to load
+# each sample defined in the dataset, and names for the various columns of the
+# CSV file. Once created, this object can be called to generate sample lists.
+from bob.ip.binseg.data.dataset import CSVDataset
+
+_raw_dataset = CSVDataset(
+ # path to the CSV file(s) - you may add as many subsets as you want:
+ # * "__train__" is used for training a model (stock data augmentation is
+ # applied via our "make_dataset()" connector)
+ # * anything else can be used for prediction and/or evaluation (if labels
+ # are also provided in such a set). Data augmentation is NOT applied
+ # using our "make_dataset()" connector.
+ subsets={
+ "__train__": "<path/to/train.csv>", #applies data augmentation
+ "train": "<path/to/train.csv>", #no data augmentation, evaluate it
+ "test": "<path/to/test.csv>", #no data augmentation, evaluate it
+ },
+ fieldnames=("data", "label"), # these are the column names
+ loader=_loader,
+)
+
+# Finally, we build a connector to passes our dataset to the pytorch framework
+# so we can, for example, train and evaluate a pytorch model. The connector
+# only converts the sample lists into a standard tuple (data[, label[, mask]])
+# that is expected by our engines, after applying the (optional)
+# transformations you define.
+
+# Add/tune your (optional) transforms below - these are just examples
+# compatible with a model that requires image inputs of 544 x 544 pixels.
+from bob.ip.binseg.data.transforms import CenterCrop
+from bob.ip.binseg.configs.datasets import make_dataset as _maker
+
+#dataset = _maker(_raw_dataset.subsets(), [CenterCrop((544, 544))])
diff --git a/bob/ip/binseg/configs/datasets/drionsdb.py b/bob/ip/binseg/configs/datasets/drionsdb.py
deleted file mode 100644
index cd33c1d246755f11c217d7cbf7fcdab35b2144e6..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drionsdb.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drionsdb import Database as DRIONS
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((4,8,4,8))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIONS(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drionsdb/__init__.py b/bob/ip/binseg/configs/datasets/drionsdb/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..99c66bb8d094cc1d8276d428c845577a09ed3d01
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drionsdb/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import Pad
+ from ....data.drionsdb import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Pad((4, 8, 4, 8))])
diff --git a/bob/ip/binseg/configs/datasets/drionsdb/expert1.py b/bob/ip/binseg/configs/datasets/drionsdb/expert1.py
new file mode 100644
index 0000000000000000000000000000000000000000..29749dbc709444c7f4eab7e1fb027b5fecbdc5b3
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drionsdb/expert1.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIONS-DB for Optic Disc Segmentation (expert #1 annotations)
+
+* Configuration resolution: 416 x 608 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.drionsdb` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drionsdb import _maker
+dataset = _maker("expert1")
diff --git a/bob/ip/binseg/configs/datasets/drionsdb/expert2.py b/bob/ip/binseg/configs/datasets/drionsdb/expert2.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e89970fd91341e5be273726024ce541edaae3a3
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drionsdb/expert2.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIONS-DB for Optic Disc Segmentation (expert #2 annotations)
+
+* Configuration resolution: 416 x 608 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.drionsdb` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drionsdb import _maker
+dataset = _maker("expert2")
diff --git a/bob/ip/binseg/configs/datasets/drionsdbtest.py b/bob/ip/binseg/configs/datasets/drionsdbtest.py
deleted file mode 100644
index b65100a6c21eaa0a48f2ff2c33e05e7a17c8825c..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drionsdbtest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drionsdb import Database as DRIONS
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((4,8,4,8))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIONS(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drishtigs1/__init__.py b/bob/ip/binseg/configs/datasets/drishtigs1/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..df8f16b9ddcb4cc6a505fcba1db1b322fbda6eed
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drishtigs1/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import CenterCrop as ccrop
+ from ....data.drishtigs1 import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [ccrop((1760, 2048))])
diff --git a/bob/ip/binseg/configs/datasets/drishtigs1/cup_all.py b/bob/ip/binseg/configs/datasets/drishtigs1/cup_all.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f648f825754cab99bbaaf4f3ba661bd0d92f07c
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drishtigs1/cup_all.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRISHTI-GS1 dataset for Cup Segmentation (agreed by all annotators)
+
+* Configuration resolution: 1760 x 2048 (after center cropping)
+* Reference (includes split): [DRISHTIGS1-2014]_
+* See :py:mod:`bob.ip.binseg.data.drishtigs1` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drishtigs1 import _maker
+dataset = _maker("optic-cup-all")
diff --git a/bob/ip/binseg/configs/datasets/drishtigs1/cup_any.py b/bob/ip/binseg/configs/datasets/drishtigs1/cup_any.py
new file mode 100644
index 0000000000000000000000000000000000000000..75471c20810b951a108f6abc58567d3fa5b5db70
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drishtigs1/cup_any.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRISHTI-GS1 dataset for Cup Segmentation (agreed by any annotator)
+
+* Configuration resolution: 1760 x 2048 (after center cropping)
+* Reference (includes split): [DRISHTIGS1-2014]_
+* See :py:mod:`bob.ip.binseg.data.drishtigs1` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drishtigs1 import _maker
+dataset = _maker("optic-cup-any")
diff --git a/bob/ip/binseg/configs/datasets/drishtigs1/disc_all.py b/bob/ip/binseg/configs/datasets/drishtigs1/disc_all.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcbe21b1fa73b418bb559f001a02d1b4f4e31873
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drishtigs1/disc_all.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRISHTI-GS1 dataset for Optic Disc Segmentation (agreed by all annotators)
+
+* Configuration resolution: 1760 x 2048 (after center cropping)
+* Reference (includes split): [DRISHTIGS1-2014]_
+* See :py:mod:`bob.ip.binseg.data.drishtigs1` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drishtigs1 import _maker
+dataset = _maker("optic-disc-all")
diff --git a/bob/ip/binseg/configs/datasets/drishtigs1/disc_any.py b/bob/ip/binseg/configs/datasets/drishtigs1/disc_any.py
new file mode 100644
index 0000000000000000000000000000000000000000..51cd3d6ad308f6c8ada7f706728ed9c6dd8a616d
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drishtigs1/disc_any.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRISHTI-GS1 dataset for Optic Disc Segmentation (agreed by any annotator)
+
+* Configuration resolution: 1760 x 2048 (after center cropping)
+* Reference (includes split): [DRISHTIGS1-2014]_
+* See :py:mod:`bob.ip.binseg.data.drishtigs1` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.drishtigs1 import _maker
+dataset = _maker("optic-disc-any")
diff --git a/bob/ip/binseg/configs/datasets/dristhigs1cup.py b/bob/ip/binseg/configs/datasets/dristhigs1cup.py
deleted file mode 100644
index f7a69dadcb4010198025e7d4d44fac8aa1b06918..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/dristhigs1cup.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drishtigs1 import Database as DRISHTI
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((1760,2048))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRISHTI(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/dristhigs1cuptest.py b/bob/ip/binseg/configs/datasets/dristhigs1cuptest.py
deleted file mode 100644
index 5c2b634e1ba402ed8af801ad7203d6585ecb6b96..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/dristhigs1cuptest.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from bob.db.drishtigs1 import Database as DRISHTI
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((1760,2048))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRISHTI(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/dristhigs1od.py b/bob/ip/binseg/configs/datasets/dristhigs1od.py
deleted file mode 100644
index 0bd483c1a03b00e15bd90a32b8ec369b766605ae..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/dristhigs1od.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drishtigs1 import Database as DRISHTI
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((1760,2048))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRISHTI(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/dristhigs1odtest.py b/bob/ip/binseg/configs/datasets/dristhigs1odtest.py
deleted file mode 100644
index ab1edd6546dac52dd235369b0126e973e2b8611a..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/dristhigs1odtest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drishtigs1 import Database as DRISHTI
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((1760,2048))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRISHTI(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive.py b/bob/ip/binseg/configs/datasets/drive.py
deleted file mode 100644
index 5b6fa356b6f944f6e7e29e1f44b3dda3ec80d9a8..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((544,544))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive/__init__.py b/bob/ip/binseg/configs/datasets/drive/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa407a8065cbc39a93d7f0c0490fe83581871d36
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import CenterCrop as ccrop
+ from ....data.drive import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [ccrop((544, 544))])
diff --git a/bob/ip/binseg/configs/datasets/drive/covd.py b/bob/ip/binseg/configs/datasets/drive/covd.py
new file mode 100644
index 0000000000000000000000000000000000000000..494ca9a9960d82fb3508d6a31238ef7810b58e60
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/covd.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-DRIVE for Vessel Segmentation
+
+* Configuration resolution: 544 x 544
+
+The dataset available in this file is composed of STARE, CHASE-DB1, IOSTAR
+vessel and HRF (with annotated samples).
+
+For details on those datasets, consult:
+
+* See :py:mod:`bob.ip.binseg.data.stare`
+* See :py:mod:`bob.ip.binseg.data.chasedb1`
+* See :py:mod:`bob.ip.binseg.data.iostar`
+* See :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.data.transforms import Resize, Pad, Crop
+from bob.ip.binseg.configs.datasets import make_trainset as _maker
+
+from bob.ip.binseg.data.stare import dataset as _raw_stare
+
+_stare = _maker(
+ _raw_stare.subsets("ah")["train"],
+ [Resize(471), Pad((0, 37, 0, 36))],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.chasedb1 import dataset as _raw_chase
+
+_chase = _maker(
+ _raw_chase.subsets("first-annotator")["train"],
+ [Resize(544), Crop(0, 12, 544, 544)],
+)
+
+from bob.ip.binseg.data.iostar import dataset as _raw_iostar
+
+_iostar = _maker(_raw_iostar.subsets("vessel")["train"], [Resize(544)],)
+
+from bob.ip.binseg.data.hrf import dataset as _raw_hrf
+
+_hrf = _maker(
+ _raw_hrf.subsets("default")["train"], [Resize((363)), Pad((0, 90, 0, 91))],
+)
+
+from torch.utils.data import ConcatDataset
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _baseline
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_baseline)
+dataset["__train__"] = ConcatDataset([_stare, _chase, _iostar, _hrf])
diff --git a/bob/ip/binseg/configs/datasets/drive/default.py b/bob/ip/binseg/configs/datasets/drive/default.py
new file mode 100644
index 0000000000000000000000000000000000000000..2331734969206068bdfb32926c495662cb5d08e7
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/default.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIVE dataset for Vessel Segmentation (default protocol)
+
+* Split reference: [DRIVE-2004]_
+* This configuration resolution: 544 x 544 (center-crop)
+* See :py:mod:`bob.ip.binseg.data.drive` for dataset details
+* This dataset offers a second-annotator comparison for the test set only
+"""
+
+from bob.ip.binseg.configs.datasets.drive import _maker
+dataset = _maker("default")
+second_annotator = _maker("second-annotator")
diff --git a/bob/ip/binseg/configs/datasets/drive/mtest.py b/bob/ip/binseg/configs/datasets/drive/mtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..201e7b898cf87bda98640f3081e659450226331c
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/mtest.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIVE cross-evaluation dataset with matched resolution
+
+* Configuration resolution: 544 x 544
+"""
+
+from bob.ip.binseg.data.transforms import Resize, Pad, Crop
+from bob.ip.binseg.configs.datasets.drive.xtest import dataset as _xt
+
+dataset = {
+ "train": _xt["train"],
+ "test": _xt["test"],
+ "stare": _xt["stare"].copy([Resize(471), Pad((0, 37, 0, 36))]),
+ "chasedb1": _xt["chasedb1"].copy([Resize(544), Crop(0, 12, 544, 544)]),
+ "hrf": _xt["hrf"].copy([Resize((363)), Pad((0, 90, 0, 91))]),
+ "iostar": _xt["iostar"].copy([Resize(544)]),
+ }
diff --git a/bob/ip/binseg/configs/datasets/drive/second_annotator.py b/bob/ip/binseg/configs/datasets/drive/second_annotator.py
new file mode 100644
index 0000000000000000000000000000000000000000..5637bbc1005846683ed9848baf6654def0f86d5d
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/second_annotator.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIVE dataset for Vessel Segmentation (second annotation: test only)
+
+* Split reference: [DRIVE-2004]_
+* This configuration resolution: 544 x 544 (center-crop)
+* See :py:mod:`bob.ip.binseg.data.drive` for dataset details
+* There are **NO training samples** on this configuration
+"""
+
+from bob.ip.binseg.configs.datasets.drive import _maker
+dataset = _maker("second-annotator")
diff --git a/bob/ip/binseg/configs/datasets/drive/ssl.py b/bob/ip/binseg/configs/datasets/drive/ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..23af544342f1a48a8e83f87d9041d638f58ed6cf
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/ssl.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-DRIVE + SSL for Vessel Segmentation
+
+* Configuration resolution: 544 x 544
+
+The dataset available in this file is composed of STARE, CHASE-DB1, IOSTAR
+vessel and HRF (with annotated samples) and DRIVE "default" training set
+without labels, for training, and DRIVE's "default" test set, for evaluation.
+
+For details on datasets, consult:
+
+* :py:mod:`bob.ip.binseg.data.stare`
+* :py:mod:`bob.ip.binseg.data.drive`
+* :py:mod:`bob.ip.binseg.data.chasedb1`
+* :py:mod:`bob.ip.binseg.data.iostar`
+* :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.configs.datasets.drive.covd import dataset as _covd
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _baseline
+from bob.ip.binseg.data.utils import SSLDataset
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_covd)
+dataset["__train__"] = SSLDataset(_covd["__train__"], _baseline["__train__"])
diff --git a/bob/ip/binseg/configs/datasets/drive/xtest.py b/bob/ip/binseg/configs/datasets/drive/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..188606b2ce50482feed9624bd40cc625ac8c38b2
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/drive/xtest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIVE cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _drive
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _stare
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _chase,
+)
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _hrf
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _iostar
+
+dataset = {
+ "train": _drive["train"],
+ "test": _drive["test"],
+ "stare": _stare["test"],
+ "chasedb1": _chase["test"],
+ "hrf": _hrf["test"],
+ "iostar": _iostar["test"],
+ }
diff --git a/bob/ip/binseg/configs/datasets/drive1024.py b/bob/ip/binseg/configs/datasets/drive1024.py
deleted file mode 100644
index dae199f50dc59c194a5ada24ff8e99f9aa4fd642..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive1024.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,CenterCrop((540,540))
- ,Resize(1024)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive1024test.py b/bob/ip/binseg/configs/datasets/drive1024test.py
deleted file mode 100644
index 9e9cb3e935a63ddb45bb0bba85c56c173de6912d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive1024test.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((540,540))
- ,Resize(1024)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive1168.py b/bob/ip/binseg/configs/datasets/drive1168.py
deleted file mode 100644
index 3f0f0537e1ba67d7beb3d77af492ba8f9fc539a2..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive1168.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Crop(75,10,416,544)
- ,Pad((21,0,22,0))
- ,Resize(1168)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive608.py b/bob/ip/binseg/configs/datasets/drive608.py
deleted file mode 100644
index 65bc5e6521dcdd3bbbf66ea77a741c6270dcdd58..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive608.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,CenterCrop((470,544))
- ,Pad((10,9,10,8))
- ,Resize(608)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drive960.py b/bob/ip/binseg/configs/datasets/drive960.py
deleted file mode 100644
index ab3ac5a9d03916dce4bbb162f51455a5bb05bcba..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drive960.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,CenterCrop((544,544))
- ,Resize(960)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py b/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py
deleted file mode 100644
index 5ab57b8ad4e34d700e4667138b6111f19b4a488a..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive608 import dataset as drive
-from bob.ip.binseg.configs.datasets.chasedb1608 import dataset as chase
-from bob.ip.binseg.configs.datasets.iostarvessel608 import dataset as iostar
-from bob.ip.binseg.configs.datasets.hrf608 import dataset as hrf
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([drive,chase,iostar,hrf])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py b/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py
deleted file mode 100644
index 928452f4209d4526e66028457a3ababcd04fda8b..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive608 import dataset as drive
-from bob.ip.binseg.configs.datasets.chasedb1608 import dataset as chase
-from bob.ip.binseg.configs.datasets.iostarvessel608 import dataset as iostar
-from bob.ip.binseg.configs.datasets.hrf608 import dataset as hrf
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-import torch
-from bob.ip.binseg.data.binsegdataset import BinSegDataset, SSLBinSegDataset, UnLabeledBinSegDataset
-
-
-#### Config ####
-
-# PyTorch dataset
-labeled_dataset = torch.utils.data.ConcatDataset([drive,chase,iostar,hrf])
-
-#### Unlabeled STARE TRAIN ####
-unlabeled_transforms = Compose([
- Pad((2,1,2,2))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-starebobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-unlabeled_dataset = UnLabeledBinSegDataset(starebobdb, split='train', transform=unlabeled_transforms)
-
-# SSL Dataset
-
-dataset = SSLBinSegDataset(labeled_dataset, unlabeled_dataset)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivestarechasedb11168.py b/bob/ip/binseg/configs/datasets/drivestarechasedb11168.py
deleted file mode 100644
index 0e36eff7195de38bd2122a0bb75942a210fba1db..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestarechasedb11168.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive1168 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare1168 import dataset as stare
-from bob.ip.binseg.configs.datasets.chasedb11168 import dataset as chase
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([drive,stare,chase])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py b/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py
deleted file mode 100644
index 74628abe77801ef572cd3ad2ff379e9af9287506..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive1024 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare1024 import dataset as stare
-from bob.ip.binseg.configs.datasets.hrf1024 import dataset as hrf
-from bob.ip.binseg.configs.datasets.chasedb11024 import dataset as chase
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([drive,stare,hrf,chase])
diff --git a/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py b/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py
deleted file mode 100644
index dd9016b7c076b08cc99e875d4f5cc9dcdd3babd0..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive1024 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare1024 import dataset as stare
-from bob.ip.binseg.configs.datasets.hrf1024 import dataset as hrf
-from bob.ip.binseg.configs.datasets.chasedb11024 import dataset as chasedb
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-import torch
-from bob.ip.binseg.data.binsegdataset import BinSegDataset, SSLBinSegDataset, UnLabeledBinSegDataset
-
-
-#### Config ####
-
-# PyTorch dataset
-labeled_dataset = torch.utils.data.ConcatDataset([drive,stare,hrf,chasedb])
-
-#### Unlabeled IOSTAR Train ####
-unlabeled_transforms = Compose([
- RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-iostarbobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-unlabeled_dataset = UnLabeledBinSegDataset(iostarbobdb, split='train', transform=unlabeled_transforms)
-
-# SSL Dataset
-
-dataset = SSLBinSegDataset(labeled_dataset, unlabeled_dataset)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py b/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py
deleted file mode 100644
index 62e2972d3526fd5ff138a5ea99f0855849beda12..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive1168 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare1168 import dataset as stare
-from bob.ip.binseg.configs.datasets.chasedb11168 import dataset as chase
-from bob.ip.binseg.configs.datasets.iostarvessel1168 import dataset as iostar
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([drive,stare,chase,iostar])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py b/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py
deleted file mode 100644
index 01705e15d8752b628ed6b150cf095db08ca4eed6..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive1168 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare1168 import dataset as stare
-from bob.ip.binseg.configs.datasets.chasedb11168 import dataset as chasedb
-from bob.ip.binseg.configs.datasets.iostarvessel1168 import dataset as iostar
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-import torch
-from bob.ip.binseg.data.binsegdataset import BinSegDataset, SSLBinSegDataset, UnLabeledBinSegDataset
-
-
-#### Config ####
-
-# PyTorch dataset
-labeled_dataset = torch.utils.data.ConcatDataset([drive,stare,iostar,chasedb])
-
-#### Unlabeled HRF TRAIN ####
-unlabeled_transforms = Compose([
- RandomRotation()
- ,Crop(0,108,2336,3296)
- ,Resize((1168))
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-hrfbobdb = HRF(protocol='default')
-
-# PyTorch dataset
-unlabeled_dataset = UnLabeledBinSegDataset(hrfbobdb, split='train', transform=unlabeled_transforms)
-
-# SSL Dataset
-
-dataset = SSLBinSegDataset(labeled_dataset, unlabeled_dataset)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py b/bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
deleted file mode 100644
index 9343cf22c62a0cf9ad02d767b24ca576b00fd4d1..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive960 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare960 import dataset as stare
-from bob.ip.binseg.configs.datasets.hrf960 import dataset as hrf
-from bob.ip.binseg.configs.datasets.iostarvessel960 import dataset as iostar
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([drive,stare,hrf,iostar])
diff --git a/bob/ip/binseg/configs/datasets/drivestareiostarhrf960sslchase.py b/bob/ip/binseg/configs/datasets/drivestareiostarhrf960sslchase.py
deleted file mode 100644
index a7bd4576766259e83ec633bfac15b4dc46f0da9d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivestareiostarhrf960sslchase.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from bob.ip.binseg.configs.datasets.drive960 import dataset as drive
-from bob.ip.binseg.configs.datasets.stare960 import dataset as stare
-from bob.ip.binseg.configs.datasets.hrf960 import dataset as hrf
-from bob.ip.binseg.configs.datasets.iostarvessel960 import dataset as iostar
-from bob.db.chasedb1 import Database as CHASE
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-import torch
-from bob.ip.binseg.data.binsegdataset import BinSegDataset, SSLBinSegDataset, UnLabeledBinSegDataset
-
-
-#### Config ####
-
-# PyTorch dataset
-labeled_dataset = torch.utils.data.ConcatDataset([drive,stare,hrf,iostar])
-
-#### Unlabeled CHASE TRAIN ####
-unlabeled_transforms = Compose([
- Crop(0,18,960,960)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-chasebobdb = CHASE(protocol = 'default')
-
-# PyTorch dataset
-unlabeled_dataset = UnLabeledBinSegDataset(chasebobdb, split='train', transform=unlabeled_transforms)
-
-# SSL Dataset
-
-dataset = SSLBinSegDataset(labeled_dataset, unlabeled_dataset)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/drivetest.py b/bob/ip/binseg/configs/datasets/drivetest.py
deleted file mode 100644
index 230598dce92a39276e05dd4b4f842643428546b4..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/drivetest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop((544,544))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/hrf.py b/bob/ip/binseg/configs/datasets/hrf.py
deleted file mode 100644
index cb008f7da1736ef66085ddfb4de0335695c28779..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,108,2336,3296)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/hrf/__init__.py b/bob/ip/binseg/configs/datasets/hrf/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3efe1b8d85cb357fc5abb5dfb4ce0e4498e54b5
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker_1168(protocol):
+
+ from ....data.transforms import Crop, Resize
+ from ....data.hrf import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Crop(0, 108, 2336, 3296), Resize(1168)])
+
+def _maker(protocol):
+
+ from ....data.transforms import Crop
+ from ....data.hrf import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Crop(0, 108, 2336, 3296)])
diff --git a/bob/ip/binseg/configs/datasets/hrf/covd.py b/bob/ip/binseg/configs/datasets/hrf/covd.py
new file mode 100644
index 0000000000000000000000000000000000000000..792a005b6c08fee3b0dbab1cea07e1379c7cf056
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/covd.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-HRF for Vessel Segmentation
+
+* Configuration resolution: 1168 x 1648
+
+The dataset available in this file is composed of DRIVE STARE, CHASE-DB1, and
+IOSTAR vessel (with annotated samples).
+
+For details on those datasets, consult:
+
+* See :py:mod:`bob.ip.binseg.data.drive`
+* See :py:mod:`bob.ip.binseg.data.stare`
+* See :py:mod:`bob.ip.binseg.data.chasedb1`
+* See :py:mod:`bob.ip.binseg.data.iostar`
+"""
+
+from bob.ip.binseg.data.transforms import Crop, Pad, Resize
+from bob.ip.binseg.configs.datasets import make_trainset as _maker
+
+from bob.ip.binseg.data.drive import dataset as _raw_drive
+
+_drive = _maker(
+ _raw_drive.subsets("default")["train"],
+ [Crop(75, 10, 416, 544), Pad((21, 0, 22, 0)), Resize(1168)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.stare import dataset as _raw_stare
+
+_stare = _maker(
+ _raw_stare.subsets("ah")["train"],
+ [Crop(50, 0, 500, 705), Resize(1168), Pad((1, 0, 1, 0))],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.chasedb1 import dataset as _raw_chase
+
+_chase = _maker(
+ _raw_chase.subsets("first-annotator")["train"],
+ [Crop(140, 18, 680, 960), Resize(1168)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.iostar import dataset as _raw_iostar
+
+_iostar = _maker(
+ _raw_iostar.subsets("vessel")["train"],
+ [Crop(144, 0, 768, 1024), Pad((30, 0, 30, 0)), Resize(1168)],
+ rotation_before=True,
+)
+
+from torch.utils.data import ConcatDataset
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _baseline
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_baseline)
+dataset["__train__"] = ConcatDataset([_drive, _stare, _chase, _iostar])
diff --git a/bob/ip/binseg/configs/datasets/hrf/default.py b/bob/ip/binseg/configs/datasets/hrf/default.py
new file mode 100644
index 0000000000000000000000000000000000000000..9089e7451039644337df43e7261204d9cd79acf5
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/default.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""HRF dataset for Vessel Segmentation (default protocol)
+
+* Split reference: [ORLANDO-2017]_
+* Configuration resolution: 1168 x 1648 (about half full HRF resolution)
+* See :py:mod:`bob.ip.binseg.data.hrf` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.hrf import _maker_1168
+dataset = _maker_1168("default")
diff --git a/bob/ip/binseg/configs/datasets/hrf/default_fullres.py b/bob/ip/binseg/configs/datasets/hrf/default_fullres.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d482510300b47068982def03e916d8fb1f6356f
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/default_fullres.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""HRF dataset for Vessel Segmentation (default protocol)
+
+* Split reference: [ORLANDO-2017]_
+* Configuration resolution: 2336 x 3296 (full dataset resolution)
+* See :py:mod:`bob.ip.binseg.data.hrf` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.hrf import _maker
+dataset = _maker("default")
diff --git a/bob/ip/binseg/configs/datasets/hrf/mtest.py b/bob/ip/binseg/configs/datasets/hrf/mtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5f8f646f4000ddcd89405e379899f45ec5c56b8
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/mtest.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""HRF cross-evaluation dataset with matched resolution
+
+* Configuration resolution: 1168 x 1648
+"""
+
+from bob.ip.binseg.data.transforms import Crop, Pad, Resize
+from bob.ip.binseg.configs.datasets.hrf.xtest import dataset as _xt
+
+dataset = {
+ "train": _xt["train"],
+ "test": _xt["test"],
+ "drive": _xt["drive"].copy(
+ [Crop(75, 10, 416, 544), Pad((21, 0, 22, 0)), Resize(1168)]
+ ),
+ "stare": _xt["stare"].copy(
+ [Crop(50, 0, 500, 705), Resize(1168), Pad((1, 0, 1, 0))]
+ ),
+ "chasedb1": _xt["chasedb1"].copy([Crop(140, 18, 680, 960), Resize(1168)]),
+ "iostar": _xt["iostar"].copy(
+ [Crop(144, 0, 768, 1024), Pad((30, 0, 30, 0)), Resize(1168)]
+ ),
+}
diff --git a/bob/ip/binseg/configs/datasets/hrf/ssl.py b/bob/ip/binseg/configs/datasets/hrf/ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f6f369e2510bb39acee083c2caa3ffa74855fa6
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/ssl.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-HRF + SSL for Vessel Segmentation
+
+* Configuration resolution: 1168 x 1648
+
+The dataset available in this file is composed of DRIVE STARE, CHASE-DB1, and
+IOSTAR vessel (with annotated samples), and HRF "default" training set, without
+labels, for training, and HRF's "default" test set, for evaluation.
+
+For details on datasets, consult:
+
+* :py:mod:`bob.ip.binseg.data.stare`
+* :py:mod:`bob.ip.binseg.data.drive`
+* :py:mod:`bob.ip.binseg.data.chasedb1`
+* :py:mod:`bob.ip.binseg.data.iostar`
+* :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.configs.datasets.hrf.covd import dataset as _covd
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _baseline
+from bob.ip.binseg.data.utils import SSLDataset
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_covd)
+dataset["__train__"] = SSLDataset(_covd["__train__"], _baseline["__train__"])
diff --git a/bob/ip/binseg/configs/datasets/hrf/xtest.py b/bob/ip/binseg/configs/datasets/hrf/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f96074fb9709f38d4ce89296e242e065520676f
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/hrf/xtest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""HRF cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _drive
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _stare
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _chase,
+)
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _hrf
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _iostar
+
+dataset = {
+ "train": _hrf["train"],
+ "test": _hrf["test"],
+ "drive": _drive["test"],
+ "stare": _stare["test"],
+ "chasedb1": _chase["test"],
+ "iostar": _iostar["test"],
+ }
diff --git a/bob/ip/binseg/configs/datasets/hrf1024.py b/bob/ip/binseg/configs/datasets/hrf1024.py
deleted file mode 100644
index 48168445f5689b95a1f55ffda0633d2acdbb619a..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf1024.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((0,584,0,584))
- ,Resize((1024))
- ,RandomRotation()
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/hrf1168.py b/bob/ip/binseg/configs/datasets/hrf1168.py
deleted file mode 100644
index 4d0c4d9eb3097a3c918a24bd2621bb3284f41215..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf1168.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,108,2336,3296)
- ,Resize((1168))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/hrf1168test.py b/bob/ip/binseg/configs/datasets/hrf1168test.py
deleted file mode 100644
index 86014b75bd7ea428a5f48f85776189d6eeccb619..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf1168test.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,108,2336,3296)
- ,Resize((1168))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/hrf544.py b/bob/ip/binseg/configs/datasets/hrf544.py
deleted file mode 100644
index 0e2cc05152aa20e4c75ffcb464cd8fd90f89e7bd..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf544.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize((363))
- ,Pad((0,90,0,91))
- ,RandomRotation()
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/hrf544test.py b/bob/ip/binseg/configs/datasets/hrf544test.py
deleted file mode 100644
index 86da428b8f6bd0220d0b311b53ddcba098177a70..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf544test.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize((363))
- ,Pad((0,90,0,91))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/hrf608.py b/bob/ip/binseg/configs/datasets/hrf608.py
deleted file mode 100644
index b26e772a0793d043af6e59f18caab5943805d929..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf608.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((0,345,0,345))
- ,Resize(608)
- ,RandomRotation()
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/hrf960.py b/bob/ip/binseg/configs/datasets/hrf960.py
deleted file mode 100644
index dd43cf00b68e8a66e8e0ec362ec519bf3488358e..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrf960.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((0,584,0,584))
- ,Resize((960))
- ,RandomRotation()
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/hrftest.py b/bob/ip/binseg/configs/datasets/hrftest.py
deleted file mode 100644
index 45f952728f8e5ff4e335c8ea6a6d79762c166b8e..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/hrftest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.hrf import Database as HRF
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Crop(0,108,2336,3296)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = HRF(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/imagefolder.py b/bob/ip/binseg/configs/datasets/imagefolder.py
deleted file mode 100644
index efcf2879a27973e01badf328a71efa265c9fa7f7..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/imagefolder.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.imagefolder import ImageFolder
-
-#### Config ####
-
-# add your transforms below
-transforms = Compose([
- CenterCrop((544,544))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# PyTorch dataset
-path = '/path/to/dataset'
-dataset = ImageFolder(path,transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/imagefolderinference.py b/bob/ip/binseg/configs/datasets/imagefolderinference.py
deleted file mode 100644
index 634566b9c0f8c899474494651044f138e610f724..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/imagefolderinference.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.imagefolderinference import ImageFolderInference
-
-#### Config ####
-
-# add your transforms below
-transforms = Compose([
- ToRGB(),
- CenterCrop((544,544))
- ,ToTensor()
- ])
-
-# PyTorch dataset
-path = '/path/to/folder/containing/images'
-dataset = ImageFolderInference(path,transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/imagefoldertest.py b/bob/ip/binseg/configs/datasets/imagefoldertest.py
deleted file mode 100644
index 15d9b0383163cee1f3b102783ceb3b50defe5459..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/imagefoldertest.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.imagefolder import ImageFolder
-
-#### Config ####
-
-# add your transforms below
-transforms = Compose([
- CenterCrop((544,544))
- ,ToTensor()
- ])
-
-# PyTorch dataset
-path = '/path/to/testdataset'
-dataset = ImageFolder(path,transform=transforms)
diff --git a/bob/ip/binseg/configs/datasets/iostar/__init__.py b/bob/ip/binseg/configs/datasets/iostar/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb28f5d47aac2d3905a946fc31c4b1d8cb5ea7d6
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.iostar import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [])
+
diff --git a/bob/ip/binseg/configs/datasets/iostar/covd.py b/bob/ip/binseg/configs/datasets/iostar/covd.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2f054feaa64222cce354265969f9e7937638be0
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/covd.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-IOSTAR for Vessel Segmentation
+
+* Configuration resolution: 1024 x 1024
+
+The dataset available in this file is composed of DRIVE, STARE, CHASE-DB1, and
+HRF (with annotated samples).
+
+For details on those datasets, consult:
+
+* See :py:mod:`bob.ip.binseg.data.drive`
+* See :py:mod:`bob.ip.binseg.data.stare`
+* See :py:mod:`bob.ip.binseg.data.chasedb1`
+* See :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Crop, Pad, Resize
+from bob.ip.binseg.configs.datasets import make_trainset as _maker
+
+from bob.ip.binseg.data.drive import dataset as _raw_drive
+
+_drive = _maker(
+ _raw_drive.subsets("default")["train"],
+ [CenterCrop((540, 540)), Resize(1024)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.stare import dataset as _raw_stare
+
+_stare = _maker(
+ _raw_stare.subsets("ah")["train"],
+ [Pad((0, 32, 0, 32)), Resize(1024), CenterCrop(1024)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.hrf import dataset as _raw_hrf
+
+_hrf = _maker(
+ _raw_hrf.subsets("default")["train"], [Pad((0, 584, 0, 584)), Resize(1024)],
+)
+
+from bob.ip.binseg.data.chasedb1 import dataset as _raw_chase
+
+_chase = _maker(
+ _raw_chase.subsets("first-annotator")["train"],
+ [Crop(0, 18, 960, 960), Resize(1024)],
+ rotation_before=True,
+)
+
+from torch.utils.data import ConcatDataset
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _baseline
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_baseline)
+dataset["__train__"] = ConcatDataset([_drive, _stare, _hrf, _chase])
diff --git a/bob/ip/binseg/configs/datasets/iostar/optic_disc.py b/bob/ip/binseg/configs/datasets/iostar/optic_disc.py
new file mode 100644
index 0000000000000000000000000000000000000000..4cbea6aa4994889e642c259ae4e89baad7deb4bd
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/optic_disc.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""IOSTAR dataset for Optic Disc Segmentation (default protocol)
+
+* Split reference: [MEYER-2017]_
+* Configuration resolution: 1024 x 1024 (original resolution)
+* See :py:mod:`bob.ip.binseg.data.iostar` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.iostar import _maker
+dataset = _maker("optic-disc")
diff --git a/bob/ip/binseg/configs/datasets/iostar/ssl.py b/bob/ip/binseg/configs/datasets/iostar/ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..2635552ee87704cd8c370c56a22431f5faa6b151
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/ssl.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-IOSTAR + SSL for Vessel Segmentation
+
+* Configuration resolution: 1024 x 1024
+
+The dataset available in this file is composed of DRIVE STARE, HRF, and
+CHASE-DB1 (with annotated samples), and IOSTAR "vessel" training set, without
+labels, for training, and IOSTAR's "vessel" test set, for evaluation.
+
+For details on datasets, consult:
+
+* :py:mod:`bob.ip.binseg.data.stare`
+* :py:mod:`bob.ip.binseg.data.drive`
+* :py:mod:`bob.ip.binseg.data.hrf`
+* :py:mod:`bob.ip.binseg.data.chasedb1`
+* :py:mod:`bob.ip.binseg.data.iostar`
+"""
+
+from bob.ip.binseg.configs.datasets.iostar.covd import dataset as _covd
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _baseline
+from bob.ip.binseg.data.utils import SSLDataset
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_covd)
+dataset["__train__"] = SSLDataset(_covd["__train__"], _baseline["__train__"])
diff --git a/bob/ip/binseg/configs/datasets/iostar/vessel.py b/bob/ip/binseg/configs/datasets/iostar/vessel.py
new file mode 100644
index 0000000000000000000000000000000000000000..a561c74e8fd920a43ec4398966acd73922f3cc76
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/vessel.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""IOSTAR dataset for Vessel Segmentation (default protocol)
+
+* Split reference: [MEYER-2017]_
+* Configuration resolution: 1024 x 1024 (original resolution)
+* See :py:mod:`bob.ip.binseg.data.iostar` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.iostar import _maker
+dataset = _maker("vessel")
diff --git a/bob/ip/binseg/configs/datasets/iostar/vessel_mtest.py b/bob/ip/binseg/configs/datasets/iostar/vessel_mtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..c89cf204442060247639277b82a8e77affd05295
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/vessel_mtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""IOSTAR vessel cross-evaluation dataset with matched resolution
+
+* Configuration resolution: 1024 x 1024
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Crop, Pad, Resize
+from bob.ip.binseg.configs.datasets.iostar.vessel_xtest import dataset as _xt
+
+dataset = {
+ "train": _xt["train"],
+ "test": _xt["test"],
+ "drive": _xt["drive"].copy([CenterCrop((540, 540)), Resize(1024)]),
+ "stare": _xt["stare"].copy(
+ [Pad((0, 32, 0, 32)), Resize(1024), CenterCrop(1024)]
+ ),
+ "chasedb1": _xt["chasedb1"].copy([Crop(0, 18, 960, 960), Resize(1024)]),
+ "hrf": _xt["hrf"].copy([Pad((0, 584, 0, 584)), Resize(1024)]),
+}
diff --git a/bob/ip/binseg/configs/datasets/iostar/vessel_xtest.py b/bob/ip/binseg/configs/datasets/iostar/vessel_xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d6272751ee4dc916c6e46f0b55209e1263f4190
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/iostar/vessel_xtest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""IOSTAR vessel cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _drive
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _stare
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _chase,
+)
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _hrf
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _iostar
+
+dataset = {
+ "train": _iostar["train"],
+ "test": _iostar["test"],
+ "drive": _drive["test"],
+ "stare": _stare["test"],
+ "chasedb1": _chase["test"],
+ "hrf": _hrf["test"],
+ }
diff --git a/bob/ip/binseg/configs/datasets/iostarod.py b/bob/ip/binseg/configs/datasets/iostarod.py
deleted file mode 100644
index 334df2a4ba402f879a436ce0ee3bcc07ca4ff49f..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarod.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarodtest.py b/bob/ip/binseg/configs/datasets/iostarodtest.py
deleted file mode 100644
index ba06450781bc03c765504e98fa715a4b15b1e774..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarodtest.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel.py b/bob/ip/binseg/configs/datasets/iostarvessel.py
deleted file mode 100644
index ded01bb45820f9402fce9a5c6dc15c14908220eb..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel1168.py b/bob/ip/binseg/configs/datasets/iostarvessel1168.py
deleted file mode 100644
index 5da5ed1e912065ca4ea2a81e4bd0f8b4e8d5475d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel1168.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Crop(144,0,768,1024)
- ,Pad((30,0,30,0))
- ,Resize(1168)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel544.py b/bob/ip/binseg/configs/datasets/iostarvessel544.py
deleted file mode 100644
index aa03abe2feb64a084fb76e6f60c69afc7499961f..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel544.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize(544)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel544test.py b/bob/ip/binseg/configs/datasets/iostarvessel544test.py
deleted file mode 100644
index e3ccd854079e57c642669aa33f403d4ba28d4700..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel544test.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize(544)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel608.py b/bob/ip/binseg/configs/datasets/iostarvessel608.py
deleted file mode 100644
index 7fce4507cd090555a2e0bac7f575dfd9d9f85c3d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel608.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((81,0,81,0))
- ,Resize(608)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvessel960.py b/bob/ip/binseg/configs/datasets/iostarvessel960.py
deleted file mode 100644
index 32feec853882cbdcadc9fea91de4a1d61e168cc0..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvessel960.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize(960)
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/iostarvesseltest.py b/bob/ip/binseg/configs/datasets/iostarvesseltest.py
deleted file mode 100644
index d8fe13718be5c4517c69683696965cbbe5a9abdf..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/iostarvesseltest.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.iostar import Database as IOSTAR
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = IOSTAR(protocol='default_vessel')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/refuge/__init__.py b/bob/ip/binseg/configs/datasets/refuge/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..24166450112b511f1847f65b275ddcfec8814437
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/refuge/__init__.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import Pad, Resize, CenterCrop
+ from ....data.refuge import dataset as raw
+ from .. import make_dataset as mk
+ # due to different sizes, we need to make the dataset twice
+ train = mk(raw.subsets(protocol), [Resize(1539), Pad((21, 46, 22, 47))])
+ # we'll keep "dev" and "test" from the next one
+ retval = mk(raw.subsets(protocol), [CenterCrop(1632)])
+ # and we keep the "train" set with the right transforms
+ retval["train"] = train["train"]
+ return retval
diff --git a/bob/ip/binseg/configs/datasets/refuge/cup.py b/bob/ip/binseg/configs/datasets/refuge/cup.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f8aef4fba1bb710ed42fc921a653491e9ce604d
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/refuge/cup.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""REFUGE dataset for Optic Cup Segmentation (default protocol)
+
+* Configuration resolution: 1632 x 1632 (after resizing and padding)
+* Reference (including split): [REFUGE-2018]_
+* See :py:mod:`bob.ip.binseg.data.refuge` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.refuge import _maker
+dataset = _maker("optic-cup")
diff --git a/bob/ip/binseg/configs/datasets/refuge/disc.py b/bob/ip/binseg/configs/datasets/refuge/disc.py
new file mode 100644
index 0000000000000000000000000000000000000000..73aa1f9276f965913343d37ae178e9c62a3344af
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/refuge/disc.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""REFUGE dataset for Optic Disc Segmentation (default protocol)
+
+* Configuration resolution: 1632 x 1632 (after resizing and padding)
+* Reference (including split): [REFUGE-2018]_
+* See :py:mod:`bob.ip.binseg.data.refuge` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.refuge import _maker
+dataset = _maker("optic-disc")
diff --git a/bob/ip/binseg/configs/datasets/refugecup.py b/bob/ip/binseg/configs/datasets/refugecup.py
deleted file mode 100644
index 9efac5293295bc0f0eb2a03a78ebacdb4e1615c2..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/refugecup.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.refuge import Database as REFUGE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize((1539))
- ,Pad((21,46,22,47))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = REFUGE(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/refugecuptest.py b/bob/ip/binseg/configs/datasets/refugecuptest.py
deleted file mode 100644
index 8ff916e30cc60e24e505ccb6c0b3455e97d7a9a9..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/refugecuptest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.refuge import Database as REFUGE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop(1632)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = REFUGE(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/refugeod.py b/bob/ip/binseg/configs/datasets/refugeod.py
deleted file mode 100644
index 5faaf05a9edcd7fcb4c3353dd6f9a17478233038..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/refugeod.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.refuge import Database as REFUGE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Resize((1539))
- ,Pad((21,46,22,47))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = REFUGE(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/refugeodtest.py b/bob/ip/binseg/configs/datasets/refugeodtest.py
deleted file mode 100644
index 30085a2f5450eefb12300b27677a15bf27baa8d8..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/refugeodtest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.refuge import Database as REFUGE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- CenterCrop(1632)
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = REFUGE(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/rimoner3/__init__.py b/bob/ip/binseg/configs/datasets/rimoner3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b9a9b941ba616e817c54f9500a761a0933db1e6
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/rimoner3/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol):
+
+ from ....data.transforms import Pad
+ from ....data.rimoner3 import dataset as raw
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Pad((8, 8, 8, 8))])
diff --git a/bob/ip/binseg/configs/datasets/rimoner3/cup_exp1.py b/bob/ip/binseg/configs/datasets/rimoner3/cup_exp1.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d9075cac2bc1c925230281685ac4774e3f37e72
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/rimoner3/cup_exp1.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""RIM-ONE r3 for Optic Cup Segmentation (expert #1 annotations)
+
+* Configuration resolution: 1440 x 1088 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.rimoner3` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.rimoner3 import _maker
+dataset = _maker("optic-cup-exp1")
diff --git a/bob/ip/binseg/configs/datasets/rimoner3/cup_exp2.py b/bob/ip/binseg/configs/datasets/rimoner3/cup_exp2.py
new file mode 100644
index 0000000000000000000000000000000000000000..aea08c7e26269ccc110f5ee5579ca8776e4fa25e
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/rimoner3/cup_exp2.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""RIM-ONE r3 for Optic Cup Segmentation (expert #2 annotations)
+
+* Configuration resolution: 1440 x 1088 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.rimoner3` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.rimoner3 import _maker
+dataset = _maker("optic-cup-exp2")
diff --git a/bob/ip/binseg/configs/datasets/rimoner3/disc_exp1.py b/bob/ip/binseg/configs/datasets/rimoner3/disc_exp1.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab5dfc08cc6b11b443680f67053c5aea160b9c26
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/rimoner3/disc_exp1.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""RIM-ONE r3 for Optic Disc Segmentation (expert #1 annotations)
+
+* Configuration resolution: 1440 x 1088 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.rimoner3` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.rimoner3 import _maker
+dataset = _maker("optic-disc-exp1")
diff --git a/bob/ip/binseg/configs/datasets/rimoner3/disc_exp2.py b/bob/ip/binseg/configs/datasets/rimoner3/disc_exp2.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ee258d8cff7fb4aa2b6294bd5627f81889c27d9
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/rimoner3/disc_exp2.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""RIM-ONE r3 for Optic Disc Segmentation (expert #2 annotations)
+
+* Configuration resolution: 1440 x 1088 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.rimoner3` for dataset details
+"""
+
+from bob.ip.binseg.configs.datasets.rimoner3 import _maker
+dataset = _maker("optic-disc-exp2")
diff --git a/bob/ip/binseg/configs/datasets/rimoner3cup.py b/bob/ip/binseg/configs/datasets/rimoner3cup.py
deleted file mode 100644
index 47b62ba0c521d4f4209fb6026c7aae184228fdb2..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/rimoner3cup.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.rimoner3 import Database as RIMONER3
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((8,8,8,8))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = RIMONER3(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/rimoner3cuptest.py b/bob/ip/binseg/configs/datasets/rimoner3cuptest.py
deleted file mode 100644
index 9f227be81289e12f4cfc8e63b1a76cbc1251c614..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/rimoner3cuptest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.rimoner3 import Database as RIMONER3
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((8,8,8,8))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = RIMONER3(protocol = 'default_cup')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/rimoner3od.py b/bob/ip/binseg/configs/datasets/rimoner3od.py
deleted file mode 100644
index 4905bec3cb663faed12cd85edc099f7987834657..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/rimoner3od.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.rimoner3 import Database as RIMONER3
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((8,8,8,8))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = RIMONER3(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/rimoner3odtest.py b/bob/ip/binseg/configs/datasets/rimoner3odtest.py
deleted file mode 100644
index 390f20d795323082c3aca3f6f0a2c81a5b144ba1..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/rimoner3odtest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.rimoner3 import Database as RIMONER3
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((8,8,8,8))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = RIMONER3(protocol = 'default_od')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/stare.py b/bob/ip/binseg/configs/datasets/stare.py
deleted file mode 100644
index f2c784a9ab7a75c82c11ffc5dbb32d6390fb93be..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/stare.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((2,1,2,2))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/stare/__init__.py b/bob/ip/binseg/configs/datasets/stare/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..82d1e14a7813b52dffa558924c861a42bab92760
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/__init__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+def _maker(protocol, raw=None):
+
+ from ....data.transforms import Pad
+ from ....data.stare import dataset as _raw
+ raw = raw or _raw #allows user to recreate dataset for testing purposes
+ from .. import make_dataset as mk
+ return mk(raw.subsets(protocol), [Pad((2, 1, 2, 2))])
diff --git a/bob/ip/binseg/configs/datasets/stare/ah.py b/bob/ip/binseg/configs/datasets/stare/ah.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b7cec73b37fb19f15f3ea6b94dc61a1a58b06fb
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/ah.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""STARE dataset for Vessel Segmentation (annotator AH)
+
+* Configuration resolution: 704 x 608 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.stare` for dataset details
+* This dataset offers a second-annotator comparison (using protocol "vk")
+"""
+
+from bob.ip.binseg.configs.datasets.stare import _maker
+dataset = _maker("ah")
+second_annotator = _maker("vk")
diff --git a/bob/ip/binseg/configs/datasets/stare/covd.py b/bob/ip/binseg/configs/datasets/stare/covd.py
new file mode 100644
index 0000000000000000000000000000000000000000..0abbf93441a7f70036073841786ac06b61ca6528
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/covd.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-STARE for Vessel Segmentation
+
+* Configuration resolution: 704 x 608
+
+The dataset available in this file is composed of DRIVE, CHASE-DB1, IOSTAR
+vessel and HRF (with annotated samples).
+
+For details on those datasets, consult:
+
+* See :py:mod:`bob.ip.binseg.data.drive`
+* See :py:mod:`bob.ip.binseg.data.chasedb1`
+* See :py:mod:`bob.ip.binseg.data.iostar`
+* See :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Pad, Resize
+from bob.ip.binseg.configs.datasets import make_trainset as _maker
+
+from bob.ip.binseg.data.drive import dataset as _raw_drive
+
+_drive = _maker(
+ _raw_drive.subsets("default")["train"],
+ [CenterCrop((470, 544)), Pad((10, 9, 10, 8)), Resize(608)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.chasedb1 import dataset as _raw_chase
+
+_chase = _maker(
+ _raw_chase.subsets("first-annotator")["train"],
+ [CenterCrop((829, 960)), Resize(608)],
+ rotation_before=True,
+)
+
+from bob.ip.binseg.data.iostar import dataset as _raw_iostar
+
+_iostar = _maker(
+ _raw_iostar.subsets("vessel")["train"],
+ # n.b.: not the best fit, but what was there for Tim's work
+ [Pad((81, 0, 81, 0)), Resize(608)],
+)
+
+from bob.ip.binseg.data.hrf import dataset as _raw_hrf
+
+_hrf = _maker(
+ _raw_hrf.subsets("default")["train"], [Pad((0, 345, 0, 345)), Resize(608)],
+)
+
+from torch.utils.data import ConcatDataset
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _baseline
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_baseline)
+dataset["__train__"] = ConcatDataset([_drive, _chase, _iostar, _hrf])
diff --git a/bob/ip/binseg/configs/datasets/stare/mtest.py b/bob/ip/binseg/configs/datasets/stare/mtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..4485ff815404e09e306aa47c7f8c1f560b877227
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/mtest.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""STARE cross-evaluation dataset with matched resolution
+
+* Configuration resolution: 704 x 608
+"""
+
+from bob.ip.binseg.data.transforms import CenterCrop, Pad, Resize
+from bob.ip.binseg.configs.datasets.stare.xtest import dataset as _xt
+
+dataset = {
+ "train": _xt["train"],
+ "test": _xt["test"],
+ "drive": _xt["drive"].copy(
+ [CenterCrop((470, 544)), Pad((10, 9, 10, 8)), Resize(608)]
+ ),
+ "chasedb1": _xt["chasedb1"].copy([CenterCrop((829, 960)), Resize(608)]),
+ "hrf": _xt["hrf"].copy([Pad((0, 345, 0, 345)), Resize(608)]),
+ "iostar": _xt["iostar"].copy([Pad((81, 0, 81, 0)), Resize(608)]),
+}
diff --git a/bob/ip/binseg/configs/datasets/stare/ssl.py b/bob/ip/binseg/configs/datasets/stare/ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..10440efd9bb35db7b499c860772c272b01bcc1f6
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/ssl.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""COVD-STARE + SSL (training set) for Vessel Segmentation
+
+* Configuration resolution: 704 x 608
+
+The dataset available in this file is composed of DRIVE, CHASE-DB1, IOSTAR
+vessel and HRF (with annotated samples) and STARE's "ah" training set, without
+labels, for training, and STARE's "ah" test set, for evaluation.
+
+For details on datasets, consult:
+
+* :py:mod:`bob.ip.binseg.data.stare`
+* :py:mod:`bob.ip.binseg.data.drive`
+* :py:mod:`bob.ip.binseg.data.chasedb1`
+* :py:mod:`bob.ip.binseg.data.iostar`
+* :py:mod:`bob.ip.binseg.data.hrf`
+"""
+
+from bob.ip.binseg.configs.datasets.stare.covd import dataset as _covd
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _baseline
+from bob.ip.binseg.data.utils import SSLDataset
+
+# copy dictionary and replace only the augmented train dataset
+dataset = dict(**_covd)
+dataset["__train__"] = SSLDataset(_covd["__train__"], _baseline["__train__"])
diff --git a/bob/ip/binseg/configs/datasets/stare/vk.py b/bob/ip/binseg/configs/datasets/stare/vk.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ae41c87553b013e0de8ad4d8601c80a80be96e2
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/vk.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""STARE dataset for Vessel Segmentation (annotator VK)
+
+* Configuration resolution: 704 x 608 (after padding)
+* Split reference: [MANINIS-2016]_
+* See :py:mod:`bob.ip.binseg.data.stare` for dataset details
+* This dataset offers a second-annotator comparison (using protocol "ah")
+"""
+
+from bob.ip.binseg.configs.datasets.stare import _maker
+dataset = _maker("vk")
+second_annotator = _maker("ah")
diff --git a/bob/ip/binseg/configs/datasets/stare/xtest.py b/bob/ip/binseg/configs/datasets/stare/xtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcd773e872ac3eaeb49b2737e0e6d78c18578d55
--- /dev/null
+++ b/bob/ip/binseg/configs/datasets/stare/xtest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""STARE cross-evaluation dataset
+"""
+
+from bob.ip.binseg.configs.datasets.drive.default import dataset as _drive
+from bob.ip.binseg.configs.datasets.stare.ah import dataset as _stare
+from bob.ip.binseg.configs.datasets.chasedb1.first_annotator import (
+ dataset as _chase,
+)
+from bob.ip.binseg.configs.datasets.hrf.default import dataset as _hrf
+from bob.ip.binseg.configs.datasets.iostar.vessel import dataset as _iostar
+
+dataset = {
+ "train": _stare["train"],
+ "test": _stare["test"],
+ "drive": _drive["test"],
+ "chasedb1": _chase["test"],
+ "hrf": _hrf["test"],
+ "iostar": _iostar["test"],
+ }
diff --git a/bob/ip/binseg/configs/datasets/stare1024.py b/bob/ip/binseg/configs/datasets/stare1024.py
deleted file mode 100644
index 8f6df507b16aeffb485bc448c57cf8b21f47bda1..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/stare1024.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Pad((0,32,0,32))
- ,Resize(1024)
- ,CenterCrop(1024)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/stare1168.py b/bob/ip/binseg/configs/datasets/stare1168.py
deleted file mode 100644
index 77e934bf6b6f387105df08519932822bcb11cf09..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/stare1168.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Crop(50,0,500,705)
- ,Resize(1168)
- ,Pad((1,0,1,0))
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/stare544.py b/bob/ip/binseg/configs/datasets/stare544.py
deleted file mode 100644
index f03fcefbe6fef80ef1e6a7ec97d2b6c1df221024..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/stare544.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([ RandomRotation()
- ,Resize(471)
- ,Pad((0,37,0,36))
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/stare960.py b/bob/ip/binseg/configs/datasets/stare960.py
deleted file mode 100644
index 0d1ed7883cb746f469534ad2a29f491501e7566e..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/stare960.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- RandomRotation()
- ,Pad((0,32,0,32))
- ,Resize(960)
- ,CenterCrop(960)
- ,RandomHFlip()
- ,RandomVFlip()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='train', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py b/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py
deleted file mode 100644
index 72349b3bd6370c0b996edf90e96403df54ec27af..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from bob.ip.binseg.configs.datasets.stare544 import dataset as stare
-from bob.ip.binseg.configs.datasets.chasedb1544 import dataset as chase
-from bob.ip.binseg.configs.datasets.iostarvessel544 import dataset as iostar
-from bob.ip.binseg.configs.datasets.hrf544 import dataset as hrf
-import torch
-
-#### Config ####
-
-# PyTorch dataset
-dataset = torch.utils.data.ConcatDataset([stare,chase,hrf,iostar])
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py b/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py
deleted file mode 100644
index 3a5e3008f73cea8d1163030783d70736fad6ef9f..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from bob.ip.binseg.configs.datasets.stare544 import dataset as stare
-from bob.ip.binseg.configs.datasets.chasedb1544 import dataset as chase
-from bob.ip.binseg.configs.datasets.iostarvessel544 import dataset as iostar
-from bob.ip.binseg.configs.datasets.hrf544 import dataset as hrf
-from bob.db.drive import Database as DRIVE
-from bob.ip.binseg.data.transforms import *
-import torch
-from bob.ip.binseg.data.binsegdataset import BinSegDataset, SSLBinSegDataset, UnLabeledBinSegDataset
-
-
-#### Config ####
-
-# PyTorch dataset
-labeled_dataset = torch.utils.data.ConcatDataset([stare,chase,iostar,hrf])
-
-#### Unlabeled STARE TRAIN ####
-unlabeled_transforms = Compose([
- CenterCrop((544,544))
- ,RandomHFlip()
- ,RandomVFlip()
- ,RandomRotation()
- ,ColorJitter()
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-drivebobdb = DRIVE(protocol = 'default')
-
-# PyTorch dataset
-unlabeled_dataset = UnLabeledBinSegDataset(drivebobdb, split='train', transform=unlabeled_transforms)
-
-# SSL Dataset
-
-dataset = SSLBinSegDataset(labeled_dataset, unlabeled_dataset)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/datasets/staretest.py b/bob/ip/binseg/configs/datasets/staretest.py
deleted file mode 100644
index aab80b9bea6339bff87c5cc12d7375ce6216bc60..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/datasets/staretest.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from bob.db.stare import Database as STARE
-from bob.ip.binseg.data.transforms import *
-from bob.ip.binseg.data.binsegdataset import BinSegDataset
-
-#### Config ####
-
-transforms = Compose([
- Pad((2,1,2,2))
- ,ToTensor()
- ])
-
-# bob.db.dataset init
-bobdb = STARE(protocol = 'default')
-
-# PyTorch dataset
-dataset = BinSegDataset(bobdb, split='test', transform=transforms)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/models/__init__.py b/bob/ip/binseg/configs/models/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/configs/models/__init__.py
+++ b/bob/ip/binseg/configs/models/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/configs/models/driu.py b/bob/ip/binseg/configs/models/driu.py
index 0e8fa1328167de67c1ca55f27791e9b2eefd6efd..cdc9cb89fa3618615dbe46bba61d25f76245fd47 100644
--- a/bob/ip/binseg/configs/models/driu.py
+++ b/bob/ip/binseg/configs/models/driu.py
@@ -1,10 +1,17 @@
#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# coding=utf-8
+
+"""DRIU Network for Vessel Segmentation
+
+Deep Retinal Image Understanding (DRIU), a unified framework of retinal image
+analysis that provides both retinal vessel and optic disc segmentation using
+deep Convolutional Neural Networks (CNNs).
+
+Reference: [MANINIS-2016]_
+"""
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.driu import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
from bob.ip.binseg.engine.adabound import AdaBound
@@ -22,17 +29,22 @@ amsbound = False
scheduler_milestones = [900]
scheduler_gamma = 0.1
-# model
model = build_driu()
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
+pretrained_backbone = modelurls["vgg16"]
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-# criterion
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/driu_bn.py b/bob/ip/binseg/configs/models/driu_bn.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e3a4b3c9121b9db5ba64805da2c24d56febbf9c
--- /dev/null
+++ b/bob/ip/binseg/configs/models/driu_bn.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""DRIU Network for Vessel Segmentation with Batch Normalization
+
+Deep Retinal Image Understanding (DRIU), a unified framework of retinal image
+analysis that provides both retinal vessel and optic disc segmentation using
+deep Convolutional Neural Networks (CNNs). This implementation includes batch
+normalization as a regularization mechanism.
+
+Reference: [MANINIS-2016]_
+"""
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.driubn import build_driu
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_driu()
+
+# pretrained backbone
+pretrained_backbone = modelurls["vgg16_bn"]
+
+# optimizer
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+# criterion
+criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
+
+# scheduler
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/driu_bn_ssl.py b/bob/ip/binseg/configs/models/driu_bn_ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..a73d4ebe051001d53c3599850c18f4f7084b5ad2
--- /dev/null
+++ b/bob/ip/binseg/configs/models/driu_bn_ssl.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""DRIU Network for Vessel Segmentation using SSL and Batch Normalization
+
+Deep Retinal Image Understanding (DRIU), a unified framework of retinal image
+analysis that provides both retinal vessel and optic disc segmentation using
+deep Convolutional Neural Networks (CNNs). This version of our model includes
+a loss that is suitable for Semi-Supervised Learning (SSL). This version also
+includes batch normalization as a regularization mechanism.
+
+Reference: [MANINIS-2016]_
+"""
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.driubn import build_driu
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import MixJacLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_driu()
+
+# pretrained backbone
+pretrained_backbone = modelurls["vgg16_bn"]
+
+# optimizer
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
+# criterion
+criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
+ssl = True
+
+# scheduler
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/driuod.py b/bob/ip/binseg/configs/models/driu_od.py
similarity index 52%
rename from bob/ip/binseg/configs/models/driuod.py
rename to bob/ip/binseg/configs/models/driu_od.py
index 7ad9bb836760b19c021f05d43ea9886685ce3257..9535c89ab5195b0d3f93971f16ad8cd9d336aab7 100644
--- a/bob/ip/binseg/configs/models/driuod.py
+++ b/bob/ip/binseg/configs/models/driu_od.py
@@ -1,10 +1,17 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+"""DRIU Network for Optic Disc Segmentation
+
+Deep Retinal Image Understanding (DRIU), a unified framework of retinal image
+analysis that provides both retinal vessel and optic disc segmentation using
+deep Convolutional Neural Networks (CNNs).
+
+Reference: [MANINIS-2016]_
+"""
+
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.driuod import build_driuod
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
from bob.ip.binseg.engine.adabound import AdaBound
@@ -26,13 +33,23 @@ scheduler_gamma = 0.1
model = build_driuod()
# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
+pretrained_backbone = modelurls["vgg16"]
# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
# criterion
criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/driu_ssl.py b/bob/ip/binseg/configs/models/driu_ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..45194f6d8a98b89f176c5aa6a7b1aa8832775eb5
--- /dev/null
+++ b/bob/ip/binseg/configs/models/driu_ssl.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""DRIU Network for Vessel Segmentation using SSL
+
+Deep Retinal Image Understanding (DRIU), a unified framework of retinal image
+analysis that provides both retinal vessel and optic disc segmentation using
+deep Convolutional Neural Networks (CNNs). This version of our model includes
+a loss that is suitable for Semi-Supervised Learning (SSL).
+
+Reference: [MANINIS-2016]_
+"""
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.driu import build_driu
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import MixJacLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_driu()
+
+# pretrained backbone
+pretrained_backbone = modelurls["vgg16"]
+
+# optimizer
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
+# criterion
+criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
+ssl = True
+
+# scheduler
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/driubn.py b/bob/ip/binseg/configs/models/driubn.py
deleted file mode 100644
index 0b95501d0a61053fd74b64976f6a761255944ece..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/models/driubn.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.driubn import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [900]
-scheduler_gamma = 0.1
-
-# model
-model = build_driu()
-
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16_bn']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-# criterion
-criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/driubnssl.py b/bob/ip/binseg/configs/models/driubnssl.py
deleted file mode 100644
index 52b3a2b35272b99d5f47bae8f23d47da15990135..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/models/driubnssl.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.driubn import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import MixJacLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [900]
-scheduler_gamma = 0.1
-
-# model
-model = build_driu()
-
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16_bn']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
-# criterion
-criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/driussl.py b/bob/ip/binseg/configs/models/driussl.py
deleted file mode 100644
index 39afd4a03f956b24aeeb078c297fb026cdc369b5..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/models/driussl.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.driu import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import MixJacLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [900]
-scheduler_gamma = 0.1
-
-# model
-model = build_driu()
-
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
-# criterion
-criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/hed.py b/bob/ip/binseg/configs/models/hed.py
index eeb0e599e89c967247975456016dc35facdf805c..6a9d7e82f211b2bd6357ad1cc19095765695af66 100644
--- a/bob/ip/binseg/configs/models/hed.py
+++ b/bob/ip/binseg/configs/models/hed.py
@@ -1,9 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+
+"""HED Network for Vessel Segmentation
+
+Holistically-nested edge detection (HED), turns pixel-wise edge classification
+into image-to-image prediction by means of a deep learning model that leverages
+fully convolutional neural networks and deeply-supervised nets.
+
+Reference: [XIE-2015]_
+"""
+
+
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.hed import build_hed
-import torch.optim as optim
from bob.ip.binseg.modeling.losses import HEDSoftJaccardBCELogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.engine.adabound import AdaBound
@@ -27,13 +37,23 @@ scheduler_gamma = 0.1
model = build_hed()
# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
+pretrained_backbone = modelurls["vgg16"]
# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
# criterion
criterion = HEDSoftJaccardBCELogitsLoss(alpha=0.7)
# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/m2unet.py b/bob/ip/binseg/configs/models/m2unet.py
index b15a277966200c97b47a201ff086c7002b9655aa..2edc0372b5b34c59e5feb9baf8b63b97f547441d 100644
--- a/bob/ip/binseg/configs/models/m2unet.py
+++ b/bob/ip/binseg/configs/models/m2unet.py
@@ -1,10 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+"""MobileNetV2 U-Net Model for Vessel Segmentation
+
+The MobileNetV2 architecture is based on an inverted residual structure where
+the input and output of the residual block are thin bottleneck layers opposite
+to traditional residual models which use expanded representations in the input
+an MobileNetV2 uses lightweight depthwise convolutions to filter features in
+the intermediate expansion layer. This model implements a MobileNetV2 U-Net
+model, henceforth named M2U-Net, combining the strenghts of U-Net for medical
+segmentation applications and the speed of MobileNetV2 networks.
+
+References: [SANDLER-2018]_, [RONNEBERGER-2015]_
+"""
+
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.m2u import build_m2unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
from bob.ip.binseg.engine.adabound import AdaBound
@@ -26,14 +37,24 @@ scheduler_gamma = 0.1
model = build_m2unet()
# pretrained backbone
-pretrained_backbone = modelurls['mobilenetv2']
+pretrained_backbone = modelurls["mobilenetv2"]
# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
# criterion
criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/m2unet_ssl.py b/bob/ip/binseg/configs/models/m2unet_ssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a456d86efd9c8efbc5a623fbd6f0a75a16ecf9d
--- /dev/null
+++ b/bob/ip/binseg/configs/models/m2unet_ssl.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+"""MobileNetV2 U-Net Model for Vessel Segmentation using SSL
+
+The MobileNetV2 architecture is based on an inverted residual structure where
+the input and output of the residual block are thin bottleneck layers opposite
+to traditional residual models which use expanded representations in the input
+an MobileNetV2 uses lightweight depthwise convolutions to filter features in
+the intermediate expansion layer. This model implements a MobileNetV2 U-Net
+model, henceforth named M2U-Net, combining the strenghts of U-Net for medical
+segmentation applications and the speed of MobileNetV2 networks. This version
+of our model includes a loss that is suitable for Semi-Supervised Learning
+(SSL).
+
+References: [SANDLER-2018]_, [RONNEBERGER-2015]_
+"""
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.m2u import build_m2unet
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import MixJacLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_m2unet()
+
+# pretrained backbone
+pretrained_backbone = modelurls["mobilenetv2"]
+
+# optimizer
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
+# criterion
+criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
+ssl = True
+
+# scheduler
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/m2unetssl.py b/bob/ip/binseg/configs/models/m2unetssl.py
deleted file mode 100644
index 3497cea26b03c44b682cfd3beb167afce3015e43..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/configs/models/m2unetssl.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.m2u import build_m2unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import MixJacLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [900]
-scheduler_gamma = 0.1
-
-# model
-model = build_m2unet()
-
-# pretrained backbone
-pretrained_backbone = modelurls['mobilenetv2']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
-# criterion
-criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/resunet.py b/bob/ip/binseg/configs/models/resunet.py
index a1db473cc7f87b6847f57cc1200326d42cb8ab86..ff7e26e599294b7f52da4bb25a71f5d08205c128 100644
--- a/bob/ip/binseg/configs/models/resunet.py
+++ b/bob/ip/binseg/configs/models/resunet.py
@@ -1,10 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+"""Residual U-Net for Vessel Segmentation
+
+A semantic segmentation neural network which combines the strengths of residual
+learning and U-Net is proposed for road area extraction. The network is built
+with residual units and has similar architecture to that of U-Net. The benefits
+of this model is two-fold: first, residual units ease training of deep
+networks. Second, the rich skip connections within the network could facilitate
+information propagation, allowing us to design networks with fewer parameters
+however better performance.
+
+Reference: [ZHANG-2017]_
+"""
+
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.resunet import build_res50unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
from bob.ip.binseg.engine.adabound import AdaBound
@@ -26,14 +37,24 @@ scheduler_gamma = 0.1
model = build_res50unet()
# pretrained backbone
-pretrained_backbone = modelurls['resnet50']
+pretrained_backbone = modelurls["resnet50"]
# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
# criterion
criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/configs/models/unet.py b/bob/ip/binseg/configs/models/unet.py
index 8182c7fa089c77221b4434e272ae6e0d394b3912..ee1eddb71417c96feb8a2897b9e0079271bdf83e 100644
--- a/bob/ip/binseg/configs/models/unet.py
+++ b/bob/ip/binseg/configs/models/unet.py
@@ -1,10 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+"""U-Net for Vessel Segmentation
+
+U-Net is a convolutional neural network that was developed for biomedical image
+segmentation at the Computer Science Department of the University of Freiburg,
+Germany. The network is based on the fully convolutional network (FCN) and its
+architecture was modified and extended to work with fewer training images and
+to yield more precise segmentations.
+
+Reference: [RONNEBERGER-2015]_
+"""
+
from torch.optim.lr_scheduler import MultiStepLR
from bob.ip.binseg.modeling.unet import build_unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
from bob.ip.binseg.utils.model_zoo import modelurls
from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
from bob.ip.binseg.engine.adabound import AdaBound
@@ -26,14 +35,24 @@ scheduler_gamma = 0.1
model = build_unet()
# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
+pretrained_backbone = modelurls["vgg16"]
# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
- eps=eps, weight_decay=weight_decay, amsbound=amsbound)
-
+optimizer = AdaBound(
+ model.parameters(),
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+)
+
# criterion
criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
+scheduler = MultiStepLR(
+ optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma
+)
diff --git a/bob/ip/binseg/data/__init__.py b/bob/ip/binseg/data/__init__.py
index d776f7534f77d642a336adab27172f4096e3b023..93e77e17a122a8d228fe4844a78f81f085973b00 100644
--- a/bob/ip/binseg/data/__init__.py
+++ b/bob/ip/binseg/data/__init__.py
@@ -1,4 +1 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
-from .binsegdataset import BinSegDataset
\ No newline at end of file
+"""Data manipulation and raw dataset definitions"""
diff --git a/bob/ip/binseg/data/binsegdataset.py b/bob/ip/binseg/data/binsegdataset.py
deleted file mode 100644
index 2917203c7b530bee796431e0dbe5e7af1f85a2b9..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/data/binsegdataset.py
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from torch.utils.data import Dataset
-import random
-
-class BinSegDataset(Dataset):
- """PyTorch dataset wrapper around bob.db binary segmentation datasets.
- A transform object can be passed that will be applied to the image, ground truth and mask (if present).
- It supports indexing such that dataset[i] can be used to get ith sample.
-
- Parameters
- ----------
- bobdb : :py:mod:`bob.db.base`
- Binary segmentation bob database (e.g. bob.db.drive)
- split : str
- ``'train'`` or ``'test'``. Defaults to ``'train'``
- transform : :py:mod:`bob.ip.binseg.data.transforms`, optional
- A transform or composition of transfroms. Defaults to ``None``.
- mask : bool
- whether dataset contains masks or not
- """
- def __init__(self, bobdb, split = 'train', transform = None,index_to = None):
- if index_to:
- self.database = bobdb.samples(split)[:index_to]
- else:
- self.database = bobdb.samples(split)
- self.transform = transform
- self.split = split
-
- @property
- def mask(self):
- # check if first sample contains a mask
- return hasattr(self.database[0], 'mask')
-
- def __len__(self):
- """
- Returns
- -------
- int
- size of the dataset
- """
- return len(self.database)
-
- def __getitem__(self,index):
- """
- Parameters
- ----------
- index : int
-
- Returns
- -------
- list
- dataitem [img_name, img, gt]
- """
- img = self.database[index].img.pil_image()
- gt = self.database[index].gt.pil_image()
- img_name = self.database[index].img.basename
- sample = [img, gt]
-
- if self.transform :
- sample = self.transform(*sample)
-
- sample.insert(0,img_name)
-
- return sample
-
-
-class SSLBinSegDataset(Dataset):
- """PyTorch dataset wrapper around bob.db binary segmentation datasets.
- A transform object can be passed that will be applied to the image, ground truth and mask (if present).
- It supports indexing such that dataset[i] can be used to get ith sample.
-
- Parameters
- ----------
- labeled_dataset : :py:class:`torch.utils.data.Dataset`
- BinSegDataset with labeled samples
- unlabeled_dataset : :py:class:`torch.utils.data.Dataset`
- UnLabeledBinSegDataset with unlabeled data
- """
- def __init__(self, labeled_dataset, unlabeled_dataset):
- self.labeled_dataset = labeled_dataset
- self.unlabeled_dataset = unlabeled_dataset
-
-
- def __len__(self):
- """
- Returns
- -------
- int
- size of the dataset
- """
- return len(self.labeled_dataset)
-
- def __getitem__(self,index):
- """
- Parameters
- ----------
- index : int
-
- Returns
- -------
- list
- dataitem [img_name, img, gt, unlabeled_img_name, unlabeled_img]
- """
- sample = self.labeled_dataset[index]
- unlabeled_img_name, unlabeled_img = self.unlabeled_dataset[0]
- sample.extend([unlabeled_img_name, unlabeled_img])
- return sample
-
-
-class UnLabeledBinSegDataset(Dataset):
- # TODO: if switch to handle case were not a bob.db object but a path to a directory is used
- """PyTorch dataset wrapper around bob.db binary segmentation datasets.
- A transform object can be passed that will be applied to the image, ground truth and mask (if present).
- It supports indexing such that dataset[i] can be used to get ith sample.
-
- Parameters
- ----------
- dv : :py:mod:`bob.db.base` or str
- Binary segmentation bob database (e.g. bob.db.drive) or path to folder containing unlabeled images
- split : str
- ``'train'`` or ``'test'``. Defaults to ``'train'``
- transform : :py:mod:`bob.ip.binseg.data.transforms`, optional
- A transform or composition of transfroms. Defaults to ``None``.
- """
- def __init__(self, db, split = 'train', transform = None,index_from= None):
- if index_from:
- self.database = db.samples(split)[index_from:]
- else:
- self.database = db.samples(split)
- self.transform = transform
- self.split = split
-
- def __len__(self):
- """
- Returns
- -------
- int
- size of the dataset
- """
- return len(self.database)
-
- def __getitem__(self,index):
- """
- Parameters
- ----------
- index : int
-
- Returns
- -------
- list
- dataitem [img_name, img]
- """
- random.shuffle(self.database)
- img = self.database[index].img.pil_image()
- img_name = self.database[index].img.basename
- sample = [img]
- if self.transform :
- sample = self.transform(img)
-
- sample.insert(0,img_name)
-
- return sample
\ No newline at end of file
diff --git a/bob/ip/binseg/data/chasedb1/__init__.py b/bob/ip/binseg/data/chasedb1/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..08a59459c8c137c89d1a64787c85cc72ddaf31d1
--- /dev/null
+++ b/bob/ip/binseg/data/chasedb1/__init__.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""CHASE-DB1 dataset for Vessel Segmentation
+
+The CHASE_DB1 is a retinal vessel reference dataset acquired from multiethnic
+school children. This database is a part of the Child Heart and Health Study in
+England (CHASE), a cardiovascular health survey in 200 primary schools in
+London, Birmingham, and Leicester. The ocular imaging was carried out in
+46 schools and demonstrated associations between retinal vessel tortuosity and
+early risk factors for cardiovascular disease in over 1000 British primary
+school children of different ethnic origin. The retinal images of both of the
+eyes of each child were recorded with a hand-held Nidek NM-200-D fundus camera.
+The images were captured at 30 degrees FOV camera. The dataset of images are
+characterized by having nonuniform back-ground illumination, poor contrast of
+blood vessels as compared with the background and wider arteriolars that have a
+bright strip running down the centre known as the central vessel reflex.
+
+* Reference: [CHASEDB1-2012]_
+* Original resolution (height x width): 960 x 999
+* Split reference: [CHASEDB1-2012]_
+* Protocol ``first-annotator``:
+
+ * Training samples: 8 (including labels from annotator "1stHO")
+ * Test samples: 20 (including labels from annotator "1stHO")
+
+* Protocol ``second-annotator``:
+
+ * Training samples: 8 (including labels from annotator "2ndHO")
+ * Test samples: 20 (including labels from annotator "2ndHO")
+
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "first-annotator.json"),
+ pkg_resources.resource_filename(__name__, "second-annotator.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.chasedb1.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _raw_data_loader(sample):
+ return dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_1(os.path.join(_root_path, sample["label"])),
+ )
+
+
+def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # we returned delayed samples to avoid loading all images at once
+ return make_delayed(sample, _raw_data_loader)
+
+
+dataset = JSONDataset(
+ protocols=_protocols, fieldnames=("data", "label"), loader=_loader
+)
+"""CHASE-DB1 dataset object"""
diff --git a/bob/ip/binseg/data/chasedb1/first-annotator.json b/bob/ip/binseg/data/chasedb1/first-annotator.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7e6761b6ec86022518152a88724d754a7231802
--- /dev/null
+++ b/bob/ip/binseg/data/chasedb1/first-annotator.json
@@ -0,0 +1,118 @@
+{
+ "train": [
+ [
+ "Image_11L.jpg",
+ "Image_11L_1stHO.png"
+ ],
+ [
+ "Image_11R.jpg",
+ "Image_11R_1stHO.png"
+ ],
+ [
+ "Image_12L.jpg",
+ "Image_12L_1stHO.png"
+ ],
+ [
+ "Image_12R.jpg",
+ "Image_12R_1stHO.png"
+ ],
+ [
+ "Image_13L.jpg",
+ "Image_13L_1stHO.png"
+ ],
+ [
+ "Image_13R.jpg",
+ "Image_13R_1stHO.png"
+ ],
+ [
+ "Image_14L.jpg",
+ "Image_14L_1stHO.png"
+ ],
+ [
+ "Image_14R.jpg",
+ "Image_14R_1stHO.png"
+ ]
+ ],
+ "test": [
+ [
+ "Image_01L.jpg",
+ "Image_01L_1stHO.png"
+ ],
+ [
+ "Image_01R.jpg",
+ "Image_01R_1stHO.png"
+ ],
+ [
+ "Image_02L.jpg",
+ "Image_02L_1stHO.png"
+ ],
+ [
+ "Image_02R.jpg",
+ "Image_02R_1stHO.png"
+ ],
+ [
+ "Image_03L.jpg",
+ "Image_03L_1stHO.png"
+ ],
+ [
+ "Image_03R.jpg",
+ "Image_03R_1stHO.png"
+ ],
+ [
+ "Image_04L.jpg",
+ "Image_04L_1stHO.png"
+ ],
+ [
+ "Image_04R.jpg",
+ "Image_04R_1stHO.png"
+ ],
+ [
+ "Image_05L.jpg",
+ "Image_05L_1stHO.png"
+ ],
+ [
+ "Image_05R.jpg",
+ "Image_05R_1stHO.png"
+ ],
+ [
+ "Image_06L.jpg",
+ "Image_06L_1stHO.png"
+ ],
+ [
+ "Image_06R.jpg",
+ "Image_06R_1stHO.png"
+ ],
+ [
+ "Image_07L.jpg",
+ "Image_07L_1stHO.png"
+ ],
+ [
+ "Image_07R.jpg",
+ "Image_07R_1stHO.png"
+ ],
+ [
+ "Image_08L.jpg",
+ "Image_08L_1stHO.png"
+ ],
+ [
+ "Image_08R.jpg",
+ "Image_08R_1stHO.png"
+ ],
+ [
+ "Image_09L.jpg",
+ "Image_09L_1stHO.png"
+ ],
+ [
+ "Image_09R.jpg",
+ "Image_09R_1stHO.png"
+ ],
+ [
+ "Image_10L.jpg",
+ "Image_10L_1stHO.png"
+ ],
+ [
+ "Image_10R.jpg",
+ "Image_10R_1stHO.png"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/chasedb1/second-annotator.json b/bob/ip/binseg/data/chasedb1/second-annotator.json
new file mode 100644
index 0000000000000000000000000000000000000000..9e26e371fa1df5095c73109cef019f1c95e8b2a5
--- /dev/null
+++ b/bob/ip/binseg/data/chasedb1/second-annotator.json
@@ -0,0 +1,118 @@
+{
+ "train": [
+ [
+ "Image_11L.jpg",
+ "Image_11L_2ndHO.png"
+ ],
+ [
+ "Image_11R.jpg",
+ "Image_11R_2ndHO.png"
+ ],
+ [
+ "Image_12L.jpg",
+ "Image_12L_2ndHO.png"
+ ],
+ [
+ "Image_12R.jpg",
+ "Image_12R_2ndHO.png"
+ ],
+ [
+ "Image_13L.jpg",
+ "Image_13L_2ndHO.png"
+ ],
+ [
+ "Image_13R.jpg",
+ "Image_13R_2ndHO.png"
+ ],
+ [
+ "Image_14L.jpg",
+ "Image_14L_2ndHO.png"
+ ],
+ [
+ "Image_14R.jpg",
+ "Image_14R_2ndHO.png"
+ ]
+ ],
+ "test": [
+ [
+ "Image_01L.jpg",
+ "Image_01L_2ndHO.png"
+ ],
+ [
+ "Image_01R.jpg",
+ "Image_01R_2ndHO.png"
+ ],
+ [
+ "Image_02L.jpg",
+ "Image_02L_2ndHO.png"
+ ],
+ [
+ "Image_02R.jpg",
+ "Image_02R_2ndHO.png"
+ ],
+ [
+ "Image_03L.jpg",
+ "Image_03L_2ndHO.png"
+ ],
+ [
+ "Image_03R.jpg",
+ "Image_03R_2ndHO.png"
+ ],
+ [
+ "Image_04L.jpg",
+ "Image_04L_2ndHO.png"
+ ],
+ [
+ "Image_04R.jpg",
+ "Image_04R_2ndHO.png"
+ ],
+ [
+ "Image_05L.jpg",
+ "Image_05L_2ndHO.png"
+ ],
+ [
+ "Image_05R.jpg",
+ "Image_05R_2ndHO.png"
+ ],
+ [
+ "Image_06L.jpg",
+ "Image_06L_2ndHO.png"
+ ],
+ [
+ "Image_06R.jpg",
+ "Image_06R_2ndHO.png"
+ ],
+ [
+ "Image_07L.jpg",
+ "Image_07L_2ndHO.png"
+ ],
+ [
+ "Image_07R.jpg",
+ "Image_07R_2ndHO.png"
+ ],
+ [
+ "Image_08L.jpg",
+ "Image_08L_2ndHO.png"
+ ],
+ [
+ "Image_08R.jpg",
+ "Image_08R_2ndHO.png"
+ ],
+ [
+ "Image_09L.jpg",
+ "Image_09L_2ndHO.png"
+ ],
+ [
+ "Image_09R.jpg",
+ "Image_09R_2ndHO.png"
+ ],
+ [
+ "Image_10L.jpg",
+ "Image_10L_2ndHO.png"
+ ],
+ [
+ "Image_10R.jpg",
+ "Image_10R_2ndHO.png"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/dataset.py b/bob/ip/binseg/data/dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd6cb14069237e2935de2814c21102888c6a55e3
--- /dev/null
+++ b/bob/ip/binseg/data/dataset.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import csv
+import copy
+import json
+import pathlib
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class JSONDataset:
+ """
+ Generic multi-protocol/subset filelist dataset that yields samples
+
+ To create a new dataset, you need to provide one or more JSON formatted
+ filelists (one per protocol) with the following contents:
+
+ .. code-block:: json
+
+ {
+ "subset1": [
+ [
+ "value1",
+ "value2",
+ "value3"
+ ],
+ [
+ "value4",
+ "value5",
+ "value6"
+ ]
+ ],
+ "subset2": [
+ ]
+ }
+
+ Your dataset many contain any number of subsets, but all sample entries
+ must contain the same number of fields.
+
+
+ Parameters
+ ----------
+
+ protocols : list, dict
+ Paths to one or more JSON formatted files containing the various
+ protocols to be recognized by this dataset, or a dictionary, mapping
+ protocol names to paths (or opened file objects) of CSV files.
+ Internally, we save a dictionary where keys default to the basename of
+ paths (list input).
+
+ fieldnames : list, tuple
+ An iterable over the field names (strings) to assign to each entry in
+ the JSON file. It should have as many items as fields in each entry of
+ the JSON file.
+
+ loader : object
+ A function that receives as input, a context dictionary (with at least
+ a "protocol" and "subset" keys indicating which protocol and subset are
+ being served), and a dictionary with ``{fieldname: value}`` entries,
+ and returns an object with at least 2 attributes:
+
+ * ``key``: which must be a unique string for every sample across
+ subsets in a protocol, and
+ * ``data``: which contains the data associated witht this sample
+
+ """
+
+ def __init__(self, protocols, fieldnames, loader):
+
+ if isinstance(protocols, dict):
+ self._protocols = protocols
+ else:
+ self._protocols = dict(
+ (os.path.splitext(os.path.basename(k))[0], k) for k in protocols
+ )
+ self.fieldnames = fieldnames
+ self._loader = loader
+
+ def check(self, limit=0):
+ """For each protocol, check if all data can be correctly accessed
+
+ This function assumes each sample has a ``data`` and a ``key``
+ attribute. The ``key`` attribute should be a string, or representable
+ as such.
+
+
+ Parameters
+ ----------
+
+ limit : int
+ Maximum number of samples to check (in each protocol/subset
+ combination) in this dataset. If set to zero, then check
+ everything.
+
+
+ Returns
+ -------
+
+ errors : int
+ Number of errors found
+
+ """
+
+ logger.info(f"Checking dataset...")
+ errors = 0
+ for proto in self._protocols:
+ logger.info(f"Checking protocol '{proto}'...")
+ for name, samples in self.subsets(proto).items():
+ logger.info(f"Checking subset '{name}'...")
+ if limit:
+ logger.info(f"Checking at most first '{limit}' samples...")
+ samples = samples[:limit]
+ for pos, sample in enumerate(samples):
+ try:
+ sample.data # may trigger data loading
+ logger.info(f"{sample.key}: OK")
+ except Exception as e:
+ logger.error(
+ f"Found error loading entry {pos} in subset {name} "
+ f"of protocol {proto} from file "
+ f"'{self._protocols[proto]}': {e}"
+ )
+ errors += 1
+ except Exception as e:
+ logger.error(f"{sample.key}: {e}")
+ errors += 1
+ return errors
+
+ def subsets(self, protocol):
+ """Returns all subsets in a protocol
+
+ This method will load JSON information for a given protocol and return
+ all subsets of the given protocol after converting each entry through
+ the loader function.
+
+ Parameters
+ ----------
+
+ protocol : str
+ Name of the protocol data to load
+
+
+ Returns
+ -------
+
+ subsets : dict
+ A dictionary mapping subset names to lists of objects (respecting
+ the ``key``, ``data`` interface).
+
+ """
+
+ fileobj = self._protocols[protocol]
+ if isinstance(fileobj, (str, bytes, pathlib.Path)):
+ with open(self._protocols[protocol], "r") as f:
+ data = json.load(f)
+ else:
+ data = json.load(f)
+ fileobj.seek(0)
+
+ retval = {}
+ for subset, samples in data.items():
+ retval[subset] = [
+ self._loader(
+ dict(protocol=protocol, subset=subset, order=n),
+ dict(zip(self.fieldnames, k))
+ )
+ for n, k in enumerate(samples)
+ ]
+
+ return retval
+
+
+class CSVDataset:
+ """
+ Generic multi-subset filelist dataset that yields samples
+
+ To create a new dataset, you only need to provide a CSV formatted filelist
+ using any separator (e.g. comma, space, semi-colon) with the following
+ information:
+
+ .. code-block:: text
+
+ value1,value2,value3
+ value4,value5,value6
+ ...
+
+ Notice that all rows must have the same number of entries.
+
+ Parameters
+ ----------
+
+ subsets : list, dict
+ Paths to one or more CSV formatted files containing the various subsets
+ to be recognized by this dataset, or a dictionary, mapping subset names
+ to paths (or opened file objects) of CSV files. Internally, we save a
+ dictionary where keys default to the basename of paths (list input).
+
+ fieldnames : list, tuple
+ An iterable over the field names (strings) to assign to each column in
+ the CSV file. It should have as many items as fields in each row of
+ the CSV file(s).
+
+ loader : object
+ A function that receives as input, a context dictionary (with, at
+ least, a "subset" key indicating which subset is being served), and a
+ dictionary with ``{key: path}`` entries, and returns a dictionary with
+ the loaded data.
+
+ """
+
+ def __init__(self, subsets, fieldnames, loader):
+
+ if isinstance(subsets, dict):
+ self._subsets = subsets
+ else:
+ self._subsets = dict(
+ (os.path.splitext(os.path.basename(k))[0], k) for k in subsets
+ )
+ self.fieldnames = fieldnames
+ self._loader = loader
+
+ def check(self, limit=0):
+ """For each subset, check if all data can be correctly accessed
+
+ This function assumes each sample has a ``data`` and a ``key``
+ attribute. The ``key`` attribute should be a string, or representable
+ as such.
+
+
+ Parameters
+ ----------
+
+ limit : int
+ Maximum number of samples to check (in each protocol/subset
+ combination) in this dataset. If set to zero, then check
+ everything.
+
+
+ Returns
+ -------
+
+ errors : int
+ Number of errors found
+
+ """
+
+ logger.info(f"Checking dataset...")
+ errors = 0
+ for name in self._subsets.keys():
+ logger.info(f"Checking subset '{name}'...")
+ samples = self.samples(name)
+ if limit:
+ logger.info(f"Checking at most first '{limit}' samples...")
+ samples = samples[:limit]
+ for pos, sample in enumerate(samples):
+ try:
+ sample.data # may trigger data loading
+ logger.info(f"{sample.key}: OK")
+ except Exception as e:
+ logger.error(
+ f"Found error loading entry {pos} in subset {name} "
+ f"from file '{self._subsets[name]}': {e}"
+ )
+ errors += 1
+ return errors
+
+ def subsets(self):
+ """Returns all available subsets at once
+
+ Returns
+ -------
+
+ subsets : dict
+ A dictionary mapping subset names to lists of objects (respecting
+ the ``key``, ``data`` interface).
+
+ """
+
+ return dict((k, self.samples(k)) for k in self._subsets.keys())
+
+ def samples(self, subset):
+ """Returns all samples in a subset
+
+ This method will load CSV information for a given subset and return
+ all samples of the given subset after passing each entry through the
+ loading function.
+
+
+ Parameters
+ ----------
+
+ subset : str
+ Name of the subset data to load
+
+
+ Returns
+ -------
+
+ subset : list
+ A lists of objects (respecting the ``key``, ``data`` interface).
+
+ """
+
+ fileobj = self._subsets[subset]
+ if isinstance(fileobj, (str, bytes, pathlib.Path)):
+ with open(self._subsets[subset], newline="") as f:
+ cf = csv.reader(f)
+ samples = [k for k in cf]
+ else:
+ cf = csv.reader(fileobj)
+ samples = [k for k in cf]
+ fileobj.seek(0)
+
+ return [
+ self._loader(
+ dict(subset=subset, order=n), dict(zip(self.fieldnames, k))
+ )
+ for n, k in enumerate(samples)
+ ]
diff --git a/bob/ip/binseg/data/drionsdb/__init__.py b/bob/ip/binseg/data/drionsdb/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..92e345e3a21ec2309aa23c8f6c42e72bb4b1b4ee
--- /dev/null
+++ b/bob/ip/binseg/data/drionsdb/__init__.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIONS-DB (training set) for Optic Disc Segmentation
+
+The dataset originates from data collected from 55 patients with glaucoma
+(23.1%) and eye hypertension (76.9%), and random selected from an eye fundus
+image base belonging to the Ophthalmology Service at Miguel Servet Hospital,
+Saragossa (Spain). It contains 110 eye fundus images with a resolution of 600
+x 400. Two sets of ground-truth optic disc annotations are available. The first
+set is commonly used for training and testing. The second set acts as a "human"
+baseline.
+
+* Reference: [DRIONSDB-2008]_
+* Original resolution (height x width): 400 x 600
+* Configuration resolution: 416 x 608 (after padding)
+* Split reference: [MANINIS-2016]_
+* Protocols ``expert1`` (baseline) and ``expert2`` (human comparison):
+
+ * Training samples: 60
+ * Test samples: 50
+"""
+
+import os
+import csv
+import pkg_resources
+
+import PIL.Image
+import PIL.ImageDraw
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "expert1.json"),
+ pkg_resources.resource_filename(__name__, "expert2.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.drionsdb.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _txt_to_pil_1(fname, size):
+ """Converts DRIONS-DB annotations to image format"""
+ with open(fname, "r") as f:
+ rows = csv.reader(f, delimiter=",", quoting=csv.QUOTE_NONNUMERIC)
+ data = list(map(tuple, rows))
+
+ retval = PIL.Image.new("1", size)
+ draw = PIL.ImageDraw.ImageDraw(retval)
+ draw.polygon(data, fill="white")
+ del draw
+ return retval
+
+
+def _pad_right(img):
+ """Pads image on the right by one pixel, respects mode"""
+ retval = PIL.Image.new(img.mode, (img.size[0] + 1, img.size[1]), "black")
+ retval.paste(img, (0, 0) + img.size) # top-left pasting
+ return retval
+
+
+def _raw_data_loader(sample):
+ data = load_pil_rgb(os.path.join(_root_path, sample["data"]))
+ label = _txt_to_pil_1(os.path.join(_root_path, sample["label"]), data.size)
+ return dict(data=data, label=label,)
+
+
+def _sample_101_loader(sample):
+ # pads the image on the right side to account for a difference in
+ # resolution to other images in the dataset
+ retval = _raw_data_loader(sample)
+ retval["data"] = _pad_right(retval["data"])
+ retval["label"] = _pad_right(retval["label"])
+ return retval
+
+
+def _loader(context, sample):
+ if sample["data"].endswith("_101.jpg"):
+ return make_delayed(sample, _sample_101_loader)
+ return make_delayed(sample, _raw_data_loader)
+
+
+dataset = JSONDataset(
+ protocols=_protocols, fieldnames=("data", "label"), loader=_loader
+)
+"""DRIONSDB dataset object"""
diff --git a/bob/ip/binseg/data/drionsdb/expert1.json b/bob/ip/binseg/data/drionsdb/expert1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7486980a2cfc8974d6547ca49ead7b912a08653f
--- /dev/null
+++ b/bob/ip/binseg/data/drionsdb/expert1.json
@@ -0,0 +1,446 @@
+{
+ "train": [
+ [
+ "images/image_001.jpg",
+ "experts_anotation/anotExpert1_001.txt"
+ ],
+ [
+ "images/image_002.jpg",
+ "experts_anotation/anotExpert1_002.txt"
+ ],
+ [
+ "images/image_003.jpg",
+ "experts_anotation/anotExpert1_003.txt"
+ ],
+ [
+ "images/image_004.jpg",
+ "experts_anotation/anotExpert1_004.txt"
+ ],
+ [
+ "images/image_005.jpg",
+ "experts_anotation/anotExpert1_005.txt"
+ ],
+ [
+ "images/image_006.jpg",
+ "experts_anotation/anotExpert1_006.txt"
+ ],
+ [
+ "images/image_007.jpg",
+ "experts_anotation/anotExpert1_007.txt"
+ ],
+ [
+ "images/image_008.jpg",
+ "experts_anotation/anotExpert1_008.txt"
+ ],
+ [
+ "images/image_009.jpg",
+ "experts_anotation/anotExpert1_009.txt"
+ ],
+ [
+ "images/image_010.jpg",
+ "experts_anotation/anotExpert1_010.txt"
+ ],
+ [
+ "images/image_011.jpg",
+ "experts_anotation/anotExpert1_011.txt"
+ ],
+ [
+ "images/image_012.jpg",
+ "experts_anotation/anotExpert1_012.txt"
+ ],
+ [
+ "images/image_013.jpg",
+ "experts_anotation/anotExpert1_013.txt"
+ ],
+ [
+ "images/image_014.jpg",
+ "experts_anotation/anotExpert1_014.txt"
+ ],
+ [
+ "images/image_015.jpg",
+ "experts_anotation/anotExpert1_015.txt"
+ ],
+ [
+ "images/image_016.jpg",
+ "experts_anotation/anotExpert1_016.txt"
+ ],
+ [
+ "images/image_017.jpg",
+ "experts_anotation/anotExpert1_017.txt"
+ ],
+ [
+ "images/image_018.jpg",
+ "experts_anotation/anotExpert1_018.txt"
+ ],
+ [
+ "images/image_019.jpg",
+ "experts_anotation/anotExpert1_019.txt"
+ ],
+ [
+ "images/image_020.jpg",
+ "experts_anotation/anotExpert1_020.txt"
+ ],
+ [
+ "images/image_021.jpg",
+ "experts_anotation/anotExpert1_021.txt"
+ ],
+ [
+ "images/image_022.jpg",
+ "experts_anotation/anotExpert1_022.txt"
+ ],
+ [
+ "images/image_023.jpg",
+ "experts_anotation/anotExpert1_023.txt"
+ ],
+ [
+ "images/image_024.jpg",
+ "experts_anotation/anotExpert1_024.txt"
+ ],
+ [
+ "images/image_025.jpg",
+ "experts_anotation/anotExpert1_025.txt"
+ ],
+ [
+ "images/image_026.jpg",
+ "experts_anotation/anotExpert1_026.txt"
+ ],
+ [
+ "images/image_027.jpg",
+ "experts_anotation/anotExpert1_027.txt"
+ ],
+ [
+ "images/image_028.jpg",
+ "experts_anotation/anotExpert1_028.txt"
+ ],
+ [
+ "images/image_029.jpg",
+ "experts_anotation/anotExpert1_029.txt"
+ ],
+ [
+ "images/image_030.jpg",
+ "experts_anotation/anotExpert1_030.txt"
+ ],
+ [
+ "images/image_031.jpg",
+ "experts_anotation/anotExpert1_031.txt"
+ ],
+ [
+ "images/image_032.jpg",
+ "experts_anotation/anotExpert1_032.txt"
+ ],
+ [
+ "images/image_033.jpg",
+ "experts_anotation/anotExpert1_033.txt"
+ ],
+ [
+ "images/image_034.jpg",
+ "experts_anotation/anotExpert1_034.txt"
+ ],
+ [
+ "images/image_035.jpg",
+ "experts_anotation/anotExpert1_035.txt"
+ ],
+ [
+ "images/image_036.jpg",
+ "experts_anotation/anotExpert1_036.txt"
+ ],
+ [
+ "images/image_037.jpg",
+ "experts_anotation/anotExpert1_037.txt"
+ ],
+ [
+ "images/image_038.jpg",
+ "experts_anotation/anotExpert1_038.txt"
+ ],
+ [
+ "images/image_039.jpg",
+ "experts_anotation/anotExpert1_039.txt"
+ ],
+ [
+ "images/image_040.jpg",
+ "experts_anotation/anotExpert1_040.txt"
+ ],
+ [
+ "images/image_041.jpg",
+ "experts_anotation/anotExpert1_041.txt"
+ ],
+ [
+ "images/image_042.jpg",
+ "experts_anotation/anotExpert1_042.txt"
+ ],
+ [
+ "images/image_043.jpg",
+ "experts_anotation/anotExpert1_043.txt"
+ ],
+ [
+ "images/image_044.jpg",
+ "experts_anotation/anotExpert1_044.txt"
+ ],
+ [
+ "images/image_045.jpg",
+ "experts_anotation/anotExpert1_045.txt"
+ ],
+ [
+ "images/image_046.jpg",
+ "experts_anotation/anotExpert1_046.txt"
+ ],
+ [
+ "images/image_047.jpg",
+ "experts_anotation/anotExpert1_047.txt"
+ ],
+ [
+ "images/image_048.jpg",
+ "experts_anotation/anotExpert1_048.txt"
+ ],
+ [
+ "images/image_049.jpg",
+ "experts_anotation/anotExpert1_049.txt"
+ ],
+ [
+ "images/image_050.jpg",
+ "experts_anotation/anotExpert1_050.txt"
+ ],
+ [
+ "images/image_051.jpg",
+ "experts_anotation/anotExpert1_051.txt"
+ ],
+ [
+ "images/image_052.jpg",
+ "experts_anotation/anotExpert1_052.txt"
+ ],
+ [
+ "images/image_053.jpg",
+ "experts_anotation/anotExpert1_053.txt"
+ ],
+ [
+ "images/image_054.jpg",
+ "experts_anotation/anotExpert1_054.txt"
+ ],
+ [
+ "images/image_055.jpg",
+ "experts_anotation/anotExpert1_055.txt"
+ ],
+ [
+ "images/image_056.jpg",
+ "experts_anotation/anotExpert1_056.txt"
+ ],
+ [
+ "images/image_057.jpg",
+ "experts_anotation/anotExpert1_057.txt"
+ ],
+ [
+ "images/image_058.jpg",
+ "experts_anotation/anotExpert1_058.txt"
+ ],
+ [
+ "images/image_059.jpg",
+ "experts_anotation/anotExpert1_059.txt"
+ ],
+ [
+ "images/image_060.jpg",
+ "experts_anotation/anotExpert1_060.txt"
+ ]
+ ],
+ "test": [
+ [
+ "images/image_061.jpg",
+ "experts_anotation/anotExpert1_061.txt"
+ ],
+ [
+ "images/image_062.jpg",
+ "experts_anotation/anotExpert1_062.txt"
+ ],
+ [
+ "images/image_063.jpg",
+ "experts_anotation/anotExpert1_063.txt"
+ ],
+ [
+ "images/image_064.jpg",
+ "experts_anotation/anotExpert1_064.txt"
+ ],
+ [
+ "images/image_065.jpg",
+ "experts_anotation/anotExpert1_065.txt"
+ ],
+ [
+ "images/image_066.jpg",
+ "experts_anotation/anotExpert1_066.txt"
+ ],
+ [
+ "images/image_067.jpg",
+ "experts_anotation/anotExpert1_067.txt"
+ ],
+ [
+ "images/image_068.jpg",
+ "experts_anotation/anotExpert1_068.txt"
+ ],
+ [
+ "images/image_069.jpg",
+ "experts_anotation/anotExpert1_069.txt"
+ ],
+ [
+ "images/image_070.jpg",
+ "experts_anotation/anotExpert1_070.txt"
+ ],
+ [
+ "images/image_071.jpg",
+ "experts_anotation/anotExpert1_071.txt"
+ ],
+ [
+ "images/image_072.jpg",
+ "experts_anotation/anotExpert1_072.txt"
+ ],
+ [
+ "images/image_073.jpg",
+ "experts_anotation/anotExpert1_073.txt"
+ ],
+ [
+ "images/image_074.jpg",
+ "experts_anotation/anotExpert1_074.txt"
+ ],
+ [
+ "images/image_075.jpg",
+ "experts_anotation/anotExpert1_075.txt"
+ ],
+ [
+ "images/image_076.jpg",
+ "experts_anotation/anotExpert1_076.txt"
+ ],
+ [
+ "images/image_077.jpg",
+ "experts_anotation/anotExpert1_077.txt"
+ ],
+ [
+ "images/image_078.jpg",
+ "experts_anotation/anotExpert1_078.txt"
+ ],
+ [
+ "images/image_079.jpg",
+ "experts_anotation/anotExpert1_079.txt"
+ ],
+ [
+ "images/image_080.jpg",
+ "experts_anotation/anotExpert1_080.txt"
+ ],
+ [
+ "images/image_081.jpg",
+ "experts_anotation/anotExpert1_081.txt"
+ ],
+ [
+ "images/image_082.jpg",
+ "experts_anotation/anotExpert1_082.txt"
+ ],
+ [
+ "images/image_083.jpg",
+ "experts_anotation/anotExpert1_083.txt"
+ ],
+ [
+ "images/image_084.jpg",
+ "experts_anotation/anotExpert1_084.txt"
+ ],
+ [
+ "images/image_085.jpg",
+ "experts_anotation/anotExpert1_085.txt"
+ ],
+ [
+ "images/image_086.jpg",
+ "experts_anotation/anotExpert1_086.txt"
+ ],
+ [
+ "images/image_087.jpg",
+ "experts_anotation/anotExpert1_087.txt"
+ ],
+ [
+ "images/image_088.jpg",
+ "experts_anotation/anotExpert1_088.txt"
+ ],
+ [
+ "images/image_089.jpg",
+ "experts_anotation/anotExpert1_089.txt"
+ ],
+ [
+ "images/image_090.jpg",
+ "experts_anotation/anotExpert1_090.txt"
+ ],
+ [
+ "images/image_091.jpg",
+ "experts_anotation/anotExpert1_091.txt"
+ ],
+ [
+ "images/image_092.jpg",
+ "experts_anotation/anotExpert1_092.txt"
+ ],
+ [
+ "images/image_093.jpg",
+ "experts_anotation/anotExpert1_093.txt"
+ ],
+ [
+ "images/image_094.jpg",
+ "experts_anotation/anotExpert1_094.txt"
+ ],
+ [
+ "images/image_095.jpg",
+ "experts_anotation/anotExpert1_095.txt"
+ ],
+ [
+ "images/image_096.jpg",
+ "experts_anotation/anotExpert1_096.txt"
+ ],
+ [
+ "images/image_097.jpg",
+ "experts_anotation/anotExpert1_097.txt"
+ ],
+ [
+ "images/image_098.jpg",
+ "experts_anotation/anotExpert1_098.txt"
+ ],
+ [
+ "images/image_099.jpg",
+ "experts_anotation/anotExpert1_099.txt"
+ ],
+ [
+ "images/image_100.jpg",
+ "experts_anotation/anotExpert1_100.txt"
+ ],
+ [
+ "images/image_101.jpg",
+ "experts_anotation/anotExpert1_101.txt"
+ ],
+ [
+ "images/image_102.jpg",
+ "experts_anotation/anotExpert1_102.txt"
+ ],
+ [
+ "images/image_103.jpg",
+ "experts_anotation/anotExpert1_103.txt"
+ ],
+ [
+ "images/image_104.jpg",
+ "experts_anotation/anotExpert1_104.txt"
+ ],
+ [
+ "images/image_105.jpg",
+ "experts_anotation/anotExpert1_105.txt"
+ ],
+ [
+ "images/image_106.jpg",
+ "experts_anotation/anotExpert1_106.txt"
+ ],
+ [
+ "images/image_107.jpg",
+ "experts_anotation/anotExpert1_107.txt"
+ ],
+ [
+ "images/image_108.jpg",
+ "experts_anotation/anotExpert1_108.txt"
+ ],
+ [
+ "images/image_109.jpg",
+ "experts_anotation/anotExpert1_109.txt"
+ ],
+ [
+ "images/image_110.jpg",
+ "experts_anotation/anotExpert1_110.txt"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/drionsdb/expert2.json b/bob/ip/binseg/data/drionsdb/expert2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f3860ce0c2a822f5db5a65c1c4effd3ef8087f5
--- /dev/null
+++ b/bob/ip/binseg/data/drionsdb/expert2.json
@@ -0,0 +1,446 @@
+{
+ "train": [
+ [
+ "images/image_001.jpg",
+ "experts_anotation/anotExpert2_001.txt"
+ ],
+ [
+ "images/image_002.jpg",
+ "experts_anotation/anotExpert2_002.txt"
+ ],
+ [
+ "images/image_003.jpg",
+ "experts_anotation/anotExpert2_003.txt"
+ ],
+ [
+ "images/image_004.jpg",
+ "experts_anotation/anotExpert2_004.txt"
+ ],
+ [
+ "images/image_005.jpg",
+ "experts_anotation/anotExpert2_005.txt"
+ ],
+ [
+ "images/image_006.jpg",
+ "experts_anotation/anotExpert2_006.txt"
+ ],
+ [
+ "images/image_007.jpg",
+ "experts_anotation/anotExpert2_007.txt"
+ ],
+ [
+ "images/image_008.jpg",
+ "experts_anotation/anotExpert2_008.txt"
+ ],
+ [
+ "images/image_009.jpg",
+ "experts_anotation/anotExpert2_009.txt"
+ ],
+ [
+ "images/image_010.jpg",
+ "experts_anotation/anotExpert2_010.txt"
+ ],
+ [
+ "images/image_011.jpg",
+ "experts_anotation/anotExpert2_011.txt"
+ ],
+ [
+ "images/image_012.jpg",
+ "experts_anotation/anotExpert2_012.txt"
+ ],
+ [
+ "images/image_013.jpg",
+ "experts_anotation/anotExpert2_013.txt"
+ ],
+ [
+ "images/image_014.jpg",
+ "experts_anotation/anotExpert2_014.txt"
+ ],
+ [
+ "images/image_015.jpg",
+ "experts_anotation/anotExpert2_015.txt"
+ ],
+ [
+ "images/image_016.jpg",
+ "experts_anotation/anotExpert2_016.txt"
+ ],
+ [
+ "images/image_017.jpg",
+ "experts_anotation/anotExpert2_017.txt"
+ ],
+ [
+ "images/image_018.jpg",
+ "experts_anotation/anotExpert2_018.txt"
+ ],
+ [
+ "images/image_019.jpg",
+ "experts_anotation/anotExpert2_019.txt"
+ ],
+ [
+ "images/image_020.jpg",
+ "experts_anotation/anotExpert2_020.txt"
+ ],
+ [
+ "images/image_021.jpg",
+ "experts_anotation/anotExpert2_021.txt"
+ ],
+ [
+ "images/image_022.jpg",
+ "experts_anotation/anotExpert2_022.txt"
+ ],
+ [
+ "images/image_023.jpg",
+ "experts_anotation/anotExpert2_023.txt"
+ ],
+ [
+ "images/image_024.jpg",
+ "experts_anotation/anotExpert2_024.txt"
+ ],
+ [
+ "images/image_025.jpg",
+ "experts_anotation/anotExpert2_025.txt"
+ ],
+ [
+ "images/image_026.jpg",
+ "experts_anotation/anotExpert2_026.txt"
+ ],
+ [
+ "images/image_027.jpg",
+ "experts_anotation/anotExpert2_027.txt"
+ ],
+ [
+ "images/image_028.jpg",
+ "experts_anotation/anotExpert2_028.txt"
+ ],
+ [
+ "images/image_029.jpg",
+ "experts_anotation/anotExpert2_029.txt"
+ ],
+ [
+ "images/image_030.jpg",
+ "experts_anotation/anotExpert2_030.txt"
+ ],
+ [
+ "images/image_031.jpg",
+ "experts_anotation/anotExpert2_031.txt"
+ ],
+ [
+ "images/image_032.jpg",
+ "experts_anotation/anotExpert2_032.txt"
+ ],
+ [
+ "images/image_033.jpg",
+ "experts_anotation/anotExpert2_033.txt"
+ ],
+ [
+ "images/image_034.jpg",
+ "experts_anotation/anotExpert2_034.txt"
+ ],
+ [
+ "images/image_035.jpg",
+ "experts_anotation/anotExpert2_035.txt"
+ ],
+ [
+ "images/image_036.jpg",
+ "experts_anotation/anotExpert2_036.txt"
+ ],
+ [
+ "images/image_037.jpg",
+ "experts_anotation/anotExpert2_037.txt"
+ ],
+ [
+ "images/image_038.jpg",
+ "experts_anotation/anotExpert2_038.txt"
+ ],
+ [
+ "images/image_039.jpg",
+ "experts_anotation/anotExpert2_039.txt"
+ ],
+ [
+ "images/image_040.jpg",
+ "experts_anotation/anotExpert2_040.txt"
+ ],
+ [
+ "images/image_041.jpg",
+ "experts_anotation/anotExpert2_041.txt"
+ ],
+ [
+ "images/image_042.jpg",
+ "experts_anotation/anotExpert2_042.txt"
+ ],
+ [
+ "images/image_043.jpg",
+ "experts_anotation/anotExpert2_043.txt"
+ ],
+ [
+ "images/image_044.jpg",
+ "experts_anotation/anotExpert2_044.txt"
+ ],
+ [
+ "images/image_045.jpg",
+ "experts_anotation/anotExpert2_045.txt"
+ ],
+ [
+ "images/image_046.jpg",
+ "experts_anotation/anotExpert2_046.txt"
+ ],
+ [
+ "images/image_047.jpg",
+ "experts_anotation/anotExpert2_047.txt"
+ ],
+ [
+ "images/image_048.jpg",
+ "experts_anotation/anotExpert2_048.txt"
+ ],
+ [
+ "images/image_049.jpg",
+ "experts_anotation/anotExpert2_049.txt"
+ ],
+ [
+ "images/image_050.jpg",
+ "experts_anotation/anotExpert2_050.txt"
+ ],
+ [
+ "images/image_051.jpg",
+ "experts_anotation/anotExpert2_051.txt"
+ ],
+ [
+ "images/image_052.jpg",
+ "experts_anotation/anotExpert2_052.txt"
+ ],
+ [
+ "images/image_053.jpg",
+ "experts_anotation/anotExpert2_053.txt"
+ ],
+ [
+ "images/image_054.jpg",
+ "experts_anotation/anotExpert2_054.txt"
+ ],
+ [
+ "images/image_055.jpg",
+ "experts_anotation/anotExpert2_055.txt"
+ ],
+ [
+ "images/image_056.jpg",
+ "experts_anotation/anotExpert2_056.txt"
+ ],
+ [
+ "images/image_057.jpg",
+ "experts_anotation/anotExpert2_057.txt"
+ ],
+ [
+ "images/image_058.jpg",
+ "experts_anotation/anotExpert2_058.txt"
+ ],
+ [
+ "images/image_059.jpg",
+ "experts_anotation/anotExpert2_059.txt"
+ ],
+ [
+ "images/image_060.jpg",
+ "experts_anotation/anotExpert2_060.txt"
+ ]
+ ],
+ "test": [
+ [
+ "images/image_061.jpg",
+ "experts_anotation/anotExpert2_061.txt"
+ ],
+ [
+ "images/image_062.jpg",
+ "experts_anotation/anotExpert2_062.txt"
+ ],
+ [
+ "images/image_063.jpg",
+ "experts_anotation/anotExpert2_063.txt"
+ ],
+ [
+ "images/image_064.jpg",
+ "experts_anotation/anotExpert2_064.txt"
+ ],
+ [
+ "images/image_065.jpg",
+ "experts_anotation/anotExpert2_065.txt"
+ ],
+ [
+ "images/image_066.jpg",
+ "experts_anotation/anotExpert2_066.txt"
+ ],
+ [
+ "images/image_067.jpg",
+ "experts_anotation/anotExpert2_067.txt"
+ ],
+ [
+ "images/image_068.jpg",
+ "experts_anotation/anotExpert2_068.txt"
+ ],
+ [
+ "images/image_069.jpg",
+ "experts_anotation/anotExpert2_069.txt"
+ ],
+ [
+ "images/image_070.jpg",
+ "experts_anotation/anotExpert2_070.txt"
+ ],
+ [
+ "images/image_071.jpg",
+ "experts_anotation/anotExpert2_071.txt"
+ ],
+ [
+ "images/image_072.jpg",
+ "experts_anotation/anotExpert2_072.txt"
+ ],
+ [
+ "images/image_073.jpg",
+ "experts_anotation/anotExpert2_073.txt"
+ ],
+ [
+ "images/image_074.jpg",
+ "experts_anotation/anotExpert2_074.txt"
+ ],
+ [
+ "images/image_075.jpg",
+ "experts_anotation/anotExpert2_075.txt"
+ ],
+ [
+ "images/image_076.jpg",
+ "experts_anotation/anotExpert2_076.txt"
+ ],
+ [
+ "images/image_077.jpg",
+ "experts_anotation/anotExpert2_077.txt"
+ ],
+ [
+ "images/image_078.jpg",
+ "experts_anotation/anotExpert2_078.txt"
+ ],
+ [
+ "images/image_079.jpg",
+ "experts_anotation/anotExpert2_079.txt"
+ ],
+ [
+ "images/image_080.jpg",
+ "experts_anotation/anotExpert2_080.txt"
+ ],
+ [
+ "images/image_081.jpg",
+ "experts_anotation/anotExpert2_081.txt"
+ ],
+ [
+ "images/image_082.jpg",
+ "experts_anotation/anotExpert2_082.txt"
+ ],
+ [
+ "images/image_083.jpg",
+ "experts_anotation/anotExpert2_083.txt"
+ ],
+ [
+ "images/image_084.jpg",
+ "experts_anotation/anotExpert2_084.txt"
+ ],
+ [
+ "images/image_085.jpg",
+ "experts_anotation/anotExpert2_085.txt"
+ ],
+ [
+ "images/image_086.jpg",
+ "experts_anotation/anotExpert2_086.txt"
+ ],
+ [
+ "images/image_087.jpg",
+ "experts_anotation/anotExpert2_087.txt"
+ ],
+ [
+ "images/image_088.jpg",
+ "experts_anotation/anotExpert2_088.txt"
+ ],
+ [
+ "images/image_089.jpg",
+ "experts_anotation/anotExpert2_089.txt"
+ ],
+ [
+ "images/image_090.jpg",
+ "experts_anotation/anotExpert2_090.txt"
+ ],
+ [
+ "images/image_091.jpg",
+ "experts_anotation/anotExpert2_091.txt"
+ ],
+ [
+ "images/image_092.jpg",
+ "experts_anotation/anotExpert2_092.txt"
+ ],
+ [
+ "images/image_093.jpg",
+ "experts_anotation/anotExpert2_093.txt"
+ ],
+ [
+ "images/image_094.jpg",
+ "experts_anotation/anotExpert2_094.txt"
+ ],
+ [
+ "images/image_095.jpg",
+ "experts_anotation/anotExpert2_095.txt"
+ ],
+ [
+ "images/image_096.jpg",
+ "experts_anotation/anotExpert2_096.txt"
+ ],
+ [
+ "images/image_097.jpg",
+ "experts_anotation/anotExpert2_097.txt"
+ ],
+ [
+ "images/image_098.jpg",
+ "experts_anotation/anotExpert2_098.txt"
+ ],
+ [
+ "images/image_099.jpg",
+ "experts_anotation/anotExpert2_099.txt"
+ ],
+ [
+ "images/image_100.jpg",
+ "experts_anotation/anotExpert2_100.txt"
+ ],
+ [
+ "images/image_101.jpg",
+ "experts_anotation/anotExpert2_101.txt"
+ ],
+ [
+ "images/image_102.jpg",
+ "experts_anotation/anotExpert2_102.txt"
+ ],
+ [
+ "images/image_103.jpg",
+ "experts_anotation/anotExpert2_103.txt"
+ ],
+ [
+ "images/image_104.jpg",
+ "experts_anotation/anotExpert2_104.txt"
+ ],
+ [
+ "images/image_105.jpg",
+ "experts_anotation/anotExpert2_105.txt"
+ ],
+ [
+ "images/image_106.jpg",
+ "experts_anotation/anotExpert2_106.txt"
+ ],
+ [
+ "images/image_107.jpg",
+ "experts_anotation/anotExpert2_107.txt"
+ ],
+ [
+ "images/image_108.jpg",
+ "experts_anotation/anotExpert2_108.txt"
+ ],
+ [
+ "images/image_109.jpg",
+ "experts_anotation/anotExpert2_109.txt"
+ ],
+ [
+ "images/image_110.jpg",
+ "experts_anotation/anotExpert2_110.txt"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/drishtigs1/__init__.py b/bob/ip/binseg/data/drishtigs1/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4ec018ec819f3221b7c0e3c6278cd11341cda4f
--- /dev/null
+++ b/bob/ip/binseg/data/drishtigs1/__init__.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Drishti-GS1 for Optic Disc and Cup Segmentation
+
+Drishti-GS is a dataset meant for validation of segmenting OD, cup and
+detecting notching. The images in the Drishti-GS dataset have been collected
+and annotated by Aravind Eye hospital, Madurai, India. This dataset is of a
+single population as all subjects whose eye images are part of this dataset are
+Indians.
+
+The dataset is divided into two: a training set and a testing set of images.
+Training images (50) are provided with groundtruths for OD and Cup segmentation
+and notching information.
+
+* Reference (including train/test split): [DRISHTIGS1-2014]_
+* Original resolution (height x width): varying (min: 1749 x 2045, max: 1845 x
+ 2468)
+* Configuration resolution: 1760 x 2048 (after center cropping)
+* Protocols ``optic-disc`` and ``optic-cup``:
+ * Training: 50
+ * Test: 51
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, make_delayed
+
+_protocols = {
+ "optic-disc-all": pkg_resources.resource_filename(
+ __name__, "optic-disc.json"
+ ),
+ "optic-cup-all": pkg_resources.resource_filename(
+ __name__, "optic-cup.json"
+ ),
+ "optic-disc-any": pkg_resources.resource_filename(
+ __name__, "optic-disc.json"
+ ),
+ "optic-cup-any": pkg_resources.resource_filename(
+ __name__, "optic-cup.json"
+ ),
+}
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.drishtigs1.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _raw_data_loader_all(sample):
+ retval = dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_rgb(os.path.join(_root_path, sample["label"])).convert(
+ "L"
+ ),
+ )
+ retval["label"] = retval["label"].point(lambda p: p > 254, mode="1")
+ return retval
+
+
+def _raw_data_loader_any(sample):
+ retval = dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_rgb(os.path.join(_root_path, sample["label"])).convert(
+ "L"
+ ),
+ )
+ retval["label"] = retval["label"].point(lambda p: p > 0, mode="1")
+ return retval
+
+
+def _loader(context, sample):
+ # Drishti-GS provides softmaps of multiple annotators
+ # we threshold to get gt where all/any of the annotators overlap
+ if context["protocol"].endswith("-all"):
+ return make_delayed(sample, _raw_data_loader_all)
+ elif context["protocol"].endswith("-any"):
+ return make_delayed(sample, _raw_data_loader_any)
+ else:
+ raise RuntimeError(f"Unknown protocol {context['protocol']}")
+
+
+dataset = JSONDataset(
+ protocols=_protocols, fieldnames=("data", "label"), loader=_loader
+)
+"""Drishti-GS1 dataset object"""
diff --git a/bob/ip/binseg/data/drishtigs1/optic-cup.json b/bob/ip/binseg/data/drishtigs1/optic-cup.json
new file mode 100644
index 0000000000000000000000000000000000000000..e543e5b60b7a09b0da7f6136a1d452039c308550
--- /dev/null
+++ b/bob/ip/binseg/data/drishtigs1/optic-cup.json
@@ -0,0 +1,410 @@
+{
+ "train": [
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_002.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_002/SoftMap/drishtiGS_002_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_004.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_004/SoftMap/drishtiGS_004_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_008.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_008/SoftMap/drishtiGS_008_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_010.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_010/SoftMap/drishtiGS_010_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_012.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_012/SoftMap/drishtiGS_012_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_015.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_015/SoftMap/drishtiGS_015_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_016.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_016/SoftMap/drishtiGS_016_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_017.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_017/SoftMap/drishtiGS_017_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_018.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_018/SoftMap/drishtiGS_018_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_022.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_022/SoftMap/drishtiGS_022_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_024.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_024/SoftMap/drishtiGS_024_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_026.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_026/SoftMap/drishtiGS_026_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_031.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_031/SoftMap/drishtiGS_031_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_032.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_032/SoftMap/drishtiGS_032_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_033.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_033/SoftMap/drishtiGS_033_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_035.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_035/SoftMap/drishtiGS_035_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_036.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_036/SoftMap/drishtiGS_036_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_037.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_037/SoftMap/drishtiGS_037_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_038.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_038/SoftMap/drishtiGS_038_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_040.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_040/SoftMap/drishtiGS_040_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_041.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_041/SoftMap/drishtiGS_041_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_042.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_042/SoftMap/drishtiGS_042_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_044.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_044/SoftMap/drishtiGS_044_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_045.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_045/SoftMap/drishtiGS_045_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_046.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_046/SoftMap/drishtiGS_046_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_047.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_047/SoftMap/drishtiGS_047_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_049.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_049/SoftMap/drishtiGS_049_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_051.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_051/SoftMap/drishtiGS_051_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_057.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_057/SoftMap/drishtiGS_057_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_058.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_058/SoftMap/drishtiGS_058_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_060.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_060/SoftMap/drishtiGS_060_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_061.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_061/SoftMap/drishtiGS_061_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_062.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_062/SoftMap/drishtiGS_062_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_063.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_063/SoftMap/drishtiGS_063_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_064.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_064/SoftMap/drishtiGS_064_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_066.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_066/SoftMap/drishtiGS_066_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_068.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_068/SoftMap/drishtiGS_068_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_069.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_069/SoftMap/drishtiGS_069_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_075.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_075/SoftMap/drishtiGS_075_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_076.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_076/SoftMap/drishtiGS_076_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_080.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_080/SoftMap/drishtiGS_080_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_081.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_081/SoftMap/drishtiGS_081_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_084.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_084/SoftMap/drishtiGS_084_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_088.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_088/SoftMap/drishtiGS_088_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_089.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_089/SoftMap/drishtiGS_089_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_090.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_090/SoftMap/drishtiGS_090_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_092.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_092/SoftMap/drishtiGS_092_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_094.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_094/SoftMap/drishtiGS_094_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_098.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_098/SoftMap/drishtiGS_098_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_101.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_101/SoftMap/drishtiGS_101_cupsegSoftmap.png"
+ ]
+ ],
+ "test": [
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_001.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_001/SoftMap/drishtiGS_001_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_003.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_003/SoftMap/drishtiGS_003_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_005.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_005/SoftMap/drishtiGS_005_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_006.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_006/SoftMap/drishtiGS_006_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_007.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_007/SoftMap/drishtiGS_007_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_009.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_009/SoftMap/drishtiGS_009_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_011.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_011/SoftMap/drishtiGS_011_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_013.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_013/SoftMap/drishtiGS_013_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_014.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_014/SoftMap/drishtiGS_014_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_019.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_019/SoftMap/drishtiGS_019_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_020.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_020/SoftMap/drishtiGS_020_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_021.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_021/SoftMap/drishtiGS_021_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_023.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_023/SoftMap/drishtiGS_023_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_025.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_025/SoftMap/drishtiGS_025_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_027.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_027/SoftMap/drishtiGS_027_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_028.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_028/SoftMap/drishtiGS_028_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_029.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_029/SoftMap/drishtiGS_029_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_030.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_030/SoftMap/drishtiGS_030_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_034.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_034/SoftMap/drishtiGS_034_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_039.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_039/SoftMap/drishtiGS_039_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_043.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_043/SoftMap/drishtiGS_043_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_048.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_048/SoftMap/drishtiGS_048_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_050.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_050/SoftMap/drishtiGS_050_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_052.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_052/SoftMap/drishtiGS_052_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_053.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_053/SoftMap/drishtiGS_053_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_054.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_054/SoftMap/drishtiGS_054_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_055.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_055/SoftMap/drishtiGS_055_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_056.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_056/SoftMap/drishtiGS_056_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_059.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_059/SoftMap/drishtiGS_059_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_065.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_065/SoftMap/drishtiGS_065_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_067.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_067/SoftMap/drishtiGS_067_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_070.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_070/SoftMap/drishtiGS_070_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_071.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_071/SoftMap/drishtiGS_071_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_072.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_072/SoftMap/drishtiGS_072_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_073.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_073/SoftMap/drishtiGS_073_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_074.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_074/SoftMap/drishtiGS_074_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_077.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_077/SoftMap/drishtiGS_077_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_078.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_078/SoftMap/drishtiGS_078_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_079.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_079/SoftMap/drishtiGS_079_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_082.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_082/SoftMap/drishtiGS_082_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_083.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_083/SoftMap/drishtiGS_083_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_085.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_085/SoftMap/drishtiGS_085_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_086.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_086/SoftMap/drishtiGS_086_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_087.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_087/SoftMap/drishtiGS_087_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_091.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_091/SoftMap/drishtiGS_091_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_093.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_093/SoftMap/drishtiGS_093_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_095.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_095/SoftMap/drishtiGS_095_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_096.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_096/SoftMap/drishtiGS_096_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_097.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_097/SoftMap/drishtiGS_097_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_099.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_099/SoftMap/drishtiGS_099_cupsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_100.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_100/SoftMap/drishtiGS_100_cupsegSoftmap.png"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/drishtigs1/optic-disc.json b/bob/ip/binseg/data/drishtigs1/optic-disc.json
new file mode 100644
index 0000000000000000000000000000000000000000..250e839833d03fd54efd62fdda942677c0844081
--- /dev/null
+++ b/bob/ip/binseg/data/drishtigs1/optic-disc.json
@@ -0,0 +1,410 @@
+{
+ "train": [
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_002.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_002/SoftMap/drishtiGS_002_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_004.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_004/SoftMap/drishtiGS_004_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_008.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_008/SoftMap/drishtiGS_008_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_010.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_010/SoftMap/drishtiGS_010_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_012.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_012/SoftMap/drishtiGS_012_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_015.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_015/SoftMap/drishtiGS_015_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_016.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_016/SoftMap/drishtiGS_016_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_017.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_017/SoftMap/drishtiGS_017_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_018.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_018/SoftMap/drishtiGS_018_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_022.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_022/SoftMap/drishtiGS_022_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_024.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_024/SoftMap/drishtiGS_024_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_026.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_026/SoftMap/drishtiGS_026_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_031.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_031/SoftMap/drishtiGS_031_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_032.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_032/SoftMap/drishtiGS_032_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_033.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_033/SoftMap/drishtiGS_033_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_035.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_035/SoftMap/drishtiGS_035_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_036.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_036/SoftMap/drishtiGS_036_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_037.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_037/SoftMap/drishtiGS_037_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_038.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_038/SoftMap/drishtiGS_038_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_040.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_040/SoftMap/drishtiGS_040_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_041.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_041/SoftMap/drishtiGS_041_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_042.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_042/SoftMap/drishtiGS_042_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_044.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_044/SoftMap/drishtiGS_044_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_045.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_045/SoftMap/drishtiGS_045_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_046.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_046/SoftMap/drishtiGS_046_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_047.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_047/SoftMap/drishtiGS_047_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_049.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_049/SoftMap/drishtiGS_049_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_051.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_051/SoftMap/drishtiGS_051_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_057.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_057/SoftMap/drishtiGS_057_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_058.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_058/SoftMap/drishtiGS_058_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_060.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_060/SoftMap/drishtiGS_060_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_061.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_061/SoftMap/drishtiGS_061_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_062.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_062/SoftMap/drishtiGS_062_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_063.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_063/SoftMap/drishtiGS_063_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_064.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_064/SoftMap/drishtiGS_064_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_066.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_066/SoftMap/drishtiGS_066_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_068.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_068/SoftMap/drishtiGS_068_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_069.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_069/SoftMap/drishtiGS_069_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_075.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_075/SoftMap/drishtiGS_075_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_076.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_076/SoftMap/drishtiGS_076_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_080.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_080/SoftMap/drishtiGS_080_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_081.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_081/SoftMap/drishtiGS_081_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_084.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_084/SoftMap/drishtiGS_084_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_088.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_088/SoftMap/drishtiGS_088_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_089.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_089/SoftMap/drishtiGS_089_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_090.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_090/SoftMap/drishtiGS_090_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_092.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_092/SoftMap/drishtiGS_092_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_094.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_094/SoftMap/drishtiGS_094_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_098.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_098/SoftMap/drishtiGS_098_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Training/Images/drishtiGS_101.png",
+ "Drishti-GS1_files/Training/GT/drishtiGS_101/SoftMap/drishtiGS_101_ODsegSoftmap.png"
+ ]
+ ],
+ "test": [
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_001.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_001/SoftMap/drishtiGS_001_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_003.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_003/SoftMap/drishtiGS_003_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_005.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_005/SoftMap/drishtiGS_005_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_006.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_006/SoftMap/drishtiGS_006_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_007.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_007/SoftMap/drishtiGS_007_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_009.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_009/SoftMap/drishtiGS_009_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_011.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_011/SoftMap/drishtiGS_011_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_013.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_013/SoftMap/drishtiGS_013_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_014.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_014/SoftMap/drishtiGS_014_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_019.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_019/SoftMap/drishtiGS_019_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_020.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_020/SoftMap/drishtiGS_020_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_021.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_021/SoftMap/drishtiGS_021_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_023.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_023/SoftMap/drishtiGS_023_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_025.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_025/SoftMap/drishtiGS_025_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_027.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_027/SoftMap/drishtiGS_027_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_028.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_028/SoftMap/drishtiGS_028_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_029.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_029/SoftMap/drishtiGS_029_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_030.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_030/SoftMap/drishtiGS_030_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_034.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_034/SoftMap/drishtiGS_034_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_039.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_039/SoftMap/drishtiGS_039_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_043.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_043/SoftMap/drishtiGS_043_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_048.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_048/SoftMap/drishtiGS_048_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_050.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_050/SoftMap/drishtiGS_050_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_052.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_052/SoftMap/drishtiGS_052_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_053.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_053/SoftMap/drishtiGS_053_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_054.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_054/SoftMap/drishtiGS_054_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_055.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_055/SoftMap/drishtiGS_055_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_056.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_056/SoftMap/drishtiGS_056_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_059.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_059/SoftMap/drishtiGS_059_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_065.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_065/SoftMap/drishtiGS_065_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_067.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_067/SoftMap/drishtiGS_067_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_070.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_070/SoftMap/drishtiGS_070_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_071.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_071/SoftMap/drishtiGS_071_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_072.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_072/SoftMap/drishtiGS_072_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_073.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_073/SoftMap/drishtiGS_073_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_074.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_074/SoftMap/drishtiGS_074_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_077.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_077/SoftMap/drishtiGS_077_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_078.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_078/SoftMap/drishtiGS_078_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_079.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_079/SoftMap/drishtiGS_079_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_082.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_082/SoftMap/drishtiGS_082_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_083.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_083/SoftMap/drishtiGS_083_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_085.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_085/SoftMap/drishtiGS_085_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_086.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_086/SoftMap/drishtiGS_086_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_087.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_087/SoftMap/drishtiGS_087_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_091.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_091/SoftMap/drishtiGS_091_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_093.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_093/SoftMap/drishtiGS_093_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_095.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_095/SoftMap/drishtiGS_095_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_096.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_096/SoftMap/drishtiGS_096_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_097.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_097/SoftMap/drishtiGS_097_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_099.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_099/SoftMap/drishtiGS_099_ODsegSoftmap.png"
+ ],
+ [
+ "Drishti-GS1_files/Test/Images/drishtiGS_100.png",
+ "Drishti-GS1_files/Test/Test_GT/drishtiGS_100/SoftMap/drishtiGS_100_ODsegSoftmap.png"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/drive/__init__.py b/bob/ip/binseg/data/drive/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..59e6f44f38c98b587cd73f8891c014c077d6fa1d
--- /dev/null
+++ b/bob/ip/binseg/data/drive/__init__.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""DRIVE dataset for Vessel Segmentation
+
+The DRIVE database has been established to enable comparative studies on
+segmentation of blood vessels in retinal images.
+
+* Reference: [DRIVE-2004]_
+* Original resolution (height x width): 584 x 565
+* Split reference: [DRIVE-2004]_
+* Protocol ``default``:
+
+ * Training samples: 20 (including labels and masks)
+ * Test samples: 20 (including labels from annotator 1 and masks)
+
+* Protocol ``second-annotator``:
+
+ * Test samples: 20 (including labels from annotator 2 and masks)
+
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "default.json"),
+ pkg_resources.resource_filename(__name__, "second-annotator.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.drive.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _raw_data_loader(sample):
+ return dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_1(os.path.join(_root_path, sample["label"])),
+ mask=load_pil_1(os.path.join(_root_path, sample["mask"])),
+ )
+
+
+def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # we returned delayed samples to avoid loading all images at once
+ return make_delayed(sample, _raw_data_loader)
+
+
+dataset = JSONDataset(
+ protocols=_protocols,
+ fieldnames=("data", "label", "mask"),
+ loader=_loader,
+)
+"""DRIVE dataset object"""
diff --git a/bob/ip/binseg/data/drive/default.json b/bob/ip/binseg/data/drive/default.json
new file mode 100644
index 0000000000000000000000000000000000000000..6707e6edd93546915d7f9960f8ba85d3449a8d6f
--- /dev/null
+++ b/bob/ip/binseg/data/drive/default.json
@@ -0,0 +1,206 @@
+{
+ "train": [
+ [
+ "training/images/21_training.tif",
+ "training/1st_manual/21_manual1.gif",
+ "training/mask/21_training_mask.gif"
+ ],
+ [
+ "training/images/22_training.tif",
+ "training/1st_manual/22_manual1.gif",
+ "training/mask/22_training_mask.gif"
+ ],
+ [
+ "training/images/23_training.tif",
+ "training/1st_manual/23_manual1.gif",
+ "training/mask/23_training_mask.gif"
+ ],
+ [
+ "training/images/24_training.tif",
+ "training/1st_manual/24_manual1.gif",
+ "training/mask/24_training_mask.gif"
+ ],
+ [
+ "training/images/25_training.tif",
+ "training/1st_manual/25_manual1.gif",
+ "training/mask/25_training_mask.gif"
+ ],
+ [
+ "training/images/26_training.tif",
+ "training/1st_manual/26_manual1.gif",
+ "training/mask/26_training_mask.gif"
+ ],
+ [
+ "training/images/27_training.tif",
+ "training/1st_manual/27_manual1.gif",
+ "training/mask/27_training_mask.gif"
+ ],
+ [
+ "training/images/28_training.tif",
+ "training/1st_manual/28_manual1.gif",
+ "training/mask/28_training_mask.gif"
+ ],
+ [
+ "training/images/29_training.tif",
+ "training/1st_manual/29_manual1.gif",
+ "training/mask/29_training_mask.gif"
+ ],
+ [
+ "training/images/30_training.tif",
+ "training/1st_manual/30_manual1.gif",
+ "training/mask/30_training_mask.gif"
+ ],
+ [
+ "training/images/31_training.tif",
+ "training/1st_manual/31_manual1.gif",
+ "training/mask/31_training_mask.gif"
+ ],
+ [
+ "training/images/32_training.tif",
+ "training/1st_manual/32_manual1.gif",
+ "training/mask/32_training_mask.gif"
+ ],
+ [
+ "training/images/33_training.tif",
+ "training/1st_manual/33_manual1.gif",
+ "training/mask/33_training_mask.gif"
+ ],
+ [
+ "training/images/34_training.tif",
+ "training/1st_manual/34_manual1.gif",
+ "training/mask/34_training_mask.gif"
+ ],
+ [
+ "training/images/35_training.tif",
+ "training/1st_manual/35_manual1.gif",
+ "training/mask/35_training_mask.gif"
+ ],
+ [
+ "training/images/36_training.tif",
+ "training/1st_manual/36_manual1.gif",
+ "training/mask/36_training_mask.gif"
+ ],
+ [
+ "training/images/37_training.tif",
+ "training/1st_manual/37_manual1.gif",
+ "training/mask/37_training_mask.gif"
+ ],
+ [
+ "training/images/38_training.tif",
+ "training/1st_manual/38_manual1.gif",
+ "training/mask/38_training_mask.gif"
+ ],
+ [
+ "training/images/39_training.tif",
+ "training/1st_manual/39_manual1.gif",
+ "training/mask/39_training_mask.gif"
+ ],
+ [
+ "training/images/40_training.tif",
+ "training/1st_manual/40_manual1.gif",
+ "training/mask/40_training_mask.gif"
+ ]
+ ],
+ "test": [
+ [
+ "test/images/01_test.tif",
+ "test/1st_manual/01_manual1.gif",
+ "test/mask/01_test_mask.gif"
+ ],
+ [
+ "test/images/02_test.tif",
+ "test/1st_manual/02_manual1.gif",
+ "test/mask/02_test_mask.gif"
+ ],
+ [
+ "test/images/03_test.tif",
+ "test/1st_manual/03_manual1.gif",
+ "test/mask/03_test_mask.gif"
+ ],
+ [
+ "test/images/04_test.tif",
+ "test/1st_manual/04_manual1.gif",
+ "test/mask/04_test_mask.gif"
+ ],
+ [
+ "test/images/05_test.tif",
+ "test/1st_manual/05_manual1.gif",
+ "test/mask/05_test_mask.gif"
+ ],
+ [
+ "test/images/06_test.tif",
+ "test/1st_manual/06_manual1.gif",
+ "test/mask/06_test_mask.gif"
+ ],
+ [
+ "test/images/07_test.tif",
+ "test/1st_manual/07_manual1.gif",
+ "test/mask/07_test_mask.gif"
+ ],
+ [
+ "test/images/08_test.tif",
+ "test/1st_manual/08_manual1.gif",
+ "test/mask/08_test_mask.gif"
+ ],
+ [
+ "test/images/09_test.tif",
+ "test/1st_manual/09_manual1.gif",
+ "test/mask/09_test_mask.gif"
+ ],
+ [
+ "test/images/10_test.tif",
+ "test/1st_manual/10_manual1.gif",
+ "test/mask/10_test_mask.gif"
+ ],
+ [
+ "test/images/11_test.tif",
+ "test/1st_manual/11_manual1.gif",
+ "test/mask/11_test_mask.gif"
+ ],
+ [
+ "test/images/12_test.tif",
+ "test/1st_manual/12_manual1.gif",
+ "test/mask/12_test_mask.gif"
+ ],
+ [
+ "test/images/13_test.tif",
+ "test/1st_manual/13_manual1.gif",
+ "test/mask/13_test_mask.gif"
+ ],
+ [
+ "test/images/14_test.tif",
+ "test/1st_manual/14_manual1.gif",
+ "test/mask/14_test_mask.gif"
+ ],
+ [
+ "test/images/15_test.tif",
+ "test/1st_manual/15_manual1.gif",
+ "test/mask/15_test_mask.gif"
+ ],
+ [
+ "test/images/16_test.tif",
+ "test/1st_manual/16_manual1.gif",
+ "test/mask/16_test_mask.gif"
+ ],
+ [
+ "test/images/17_test.tif",
+ "test/1st_manual/17_manual1.gif",
+ "test/mask/17_test_mask.gif"
+ ],
+ [
+ "test/images/18_test.tif",
+ "test/1st_manual/18_manual1.gif",
+ "test/mask/18_test_mask.gif"
+ ],
+ [
+ "test/images/19_test.tif",
+ "test/1st_manual/19_manual1.gif",
+ "test/mask/19_test_mask.gif"
+ ],
+ [
+ "test/images/20_test.tif",
+ "test/1st_manual/20_manual1.gif",
+ "test/mask/20_test_mask.gif"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/drive/second-annotator.json b/bob/ip/binseg/data/drive/second-annotator.json
new file mode 100644
index 0000000000000000000000000000000000000000..fee520debd55220ccfd82145df4c70e39b1fc6b5
--- /dev/null
+++ b/bob/ip/binseg/data/drive/second-annotator.json
@@ -0,0 +1,104 @@
+{
+ "test": [
+ [
+ "test/images/01_test.tif",
+ "test/2nd_manual/01_manual2.gif",
+ "test/mask/01_test_mask.gif"
+ ],
+ [
+ "test/images/02_test.tif",
+ "test/2nd_manual/02_manual2.gif",
+ "test/mask/02_test_mask.gif"
+ ],
+ [
+ "test/images/03_test.tif",
+ "test/2nd_manual/03_manual2.gif",
+ "test/mask/03_test_mask.gif"
+ ],
+ [
+ "test/images/04_test.tif",
+ "test/2nd_manual/04_manual2.gif",
+ "test/mask/04_test_mask.gif"
+ ],
+ [
+ "test/images/05_test.tif",
+ "test/2nd_manual/05_manual2.gif",
+ "test/mask/05_test_mask.gif"
+ ],
+ [
+ "test/images/06_test.tif",
+ "test/2nd_manual/06_manual2.gif",
+ "test/mask/06_test_mask.gif"
+ ],
+ [
+ "test/images/07_test.tif",
+ "test/2nd_manual/07_manual2.gif",
+ "test/mask/07_test_mask.gif"
+ ],
+ [
+ "test/images/08_test.tif",
+ "test/2nd_manual/08_manual2.gif",
+ "test/mask/08_test_mask.gif"
+ ],
+ [
+ "test/images/09_test.tif",
+ "test/2nd_manual/09_manual2.gif",
+ "test/mask/09_test_mask.gif"
+ ],
+ [
+ "test/images/10_test.tif",
+ "test/2nd_manual/10_manual2.gif",
+ "test/mask/10_test_mask.gif"
+ ],
+ [
+ "test/images/11_test.tif",
+ "test/2nd_manual/11_manual2.gif",
+ "test/mask/11_test_mask.gif"
+ ],
+ [
+ "test/images/12_test.tif",
+ "test/2nd_manual/12_manual2.gif",
+ "test/mask/12_test_mask.gif"
+ ],
+ [
+ "test/images/13_test.tif",
+ "test/2nd_manual/13_manual2.gif",
+ "test/mask/13_test_mask.gif"
+ ],
+ [
+ "test/images/14_test.tif",
+ "test/2nd_manual/14_manual2.gif",
+ "test/mask/14_test_mask.gif"
+ ],
+ [
+ "test/images/15_test.tif",
+ "test/2nd_manual/15_manual2.gif",
+ "test/mask/15_test_mask.gif"
+ ],
+ [
+ "test/images/16_test.tif",
+ "test/2nd_manual/16_manual2.gif",
+ "test/mask/16_test_mask.gif"
+ ],
+ [
+ "test/images/17_test.tif",
+ "test/2nd_manual/17_manual2.gif",
+ "test/mask/17_test_mask.gif"
+ ],
+ [
+ "test/images/18_test.tif",
+ "test/2nd_manual/18_manual2.gif",
+ "test/mask/18_test_mask.gif"
+ ],
+ [
+ "test/images/19_test.tif",
+ "test/2nd_manual/19_manual2.gif",
+ "test/mask/19_test_mask.gif"
+ ],
+ [
+ "test/images/20_test.tif",
+ "test/2nd_manual/20_manual2.gif",
+ "test/mask/20_test_mask.gif"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/hrf/__init__.py b/bob/ip/binseg/data/hrf/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd9f8da453f3863fb20b2d1e3af58cdde958e68f
--- /dev/null
+++ b/bob/ip/binseg/data/hrf/__init__.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""HRF dataset for Vessel Segmentation
+
+The database includes 15 images of each healthy, diabetic retinopathy (DR), and
+glaucomatous eyes. It contains a total of 45 eye fundus images with a
+resolution of 3304 x 2336. One set of ground-truth vessel annotations is
+available.
+
+* Reference: [HRF-2013]_
+* Original resolution (height x width): 2336 x 3504
+* Configuration resolution: 1168 x 1648 (after specific cropping and rescaling)
+* Split reference: [ORLANDO-2017]_
+* Protocol ``default``:
+
+ * Training samples: 15 (including labels)
+ * Test samples: 30 (including labels)
+
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "default.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.hrf.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _raw_data_loader(sample):
+ return dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_1(os.path.join(_root_path, sample["label"])),
+ mask=load_pil_1(os.path.join(_root_path, sample["mask"])),
+ )
+
+
+def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # we returned delayed samples to avoid loading all images at once
+ return make_delayed(sample, _raw_data_loader)
+
+
+dataset = JSONDataset(
+ protocols=_protocols, fieldnames=("data", "label", "mask"), loader=_loader,
+)
+"""HRF dataset object"""
diff --git a/bob/ip/binseg/data/hrf/default.json b/bob/ip/binseg/data/hrf/default.json
new file mode 100644
index 0000000000000000000000000000000000000000..87f29ad575e071c2abae8118c6b99c3d19dc24b8
--- /dev/null
+++ b/bob/ip/binseg/data/hrf/default.json
@@ -0,0 +1,231 @@
+{
+ "train": [
+ [
+ "images/01_dr.JPG",
+ "manual1/01_dr.tif",
+ "mask/01_dr_mask.tif"
+ ],
+ [
+ "images/02_dr.JPG",
+ "manual1/02_dr.tif",
+ "mask/02_dr_mask.tif"
+ ],
+ [
+ "images/03_dr.JPG",
+ "manual1/03_dr.tif",
+ "mask/03_dr_mask.tif"
+ ],
+ [
+ "images/04_dr.JPG",
+ "manual1/04_dr.tif",
+ "mask/04_dr_mask.tif"
+ ],
+ [
+ "images/05_dr.JPG",
+ "manual1/05_dr.tif",
+ "mask/05_dr_mask.tif"
+ ],
+ [
+ "images/01_g.jpg",
+ "manual1/01_g.tif",
+ "mask/01_g_mask.tif"
+ ],
+ [
+ "images/02_g.jpg",
+ "manual1/02_g.tif",
+ "mask/02_g_mask.tif"
+ ],
+ [
+ "images/03_g.jpg",
+ "manual1/03_g.tif",
+ "mask/03_g_mask.tif"
+ ],
+ [
+ "images/04_g.jpg",
+ "manual1/04_g.tif",
+ "mask/04_g_mask.tif"
+ ],
+ [
+ "images/05_g.jpg",
+ "manual1/05_g.tif",
+ "mask/05_g_mask.tif"
+ ],
+ [
+ "images/01_h.jpg",
+ "manual1/01_h.tif",
+ "mask/01_h_mask.tif"
+ ],
+ [
+ "images/02_h.jpg",
+ "manual1/02_h.tif",
+ "mask/02_h_mask.tif"
+ ],
+ [
+ "images/03_h.jpg",
+ "manual1/03_h.tif",
+ "mask/03_h_mask.tif"
+ ],
+ [
+ "images/04_h.jpg",
+ "manual1/04_h.tif",
+ "mask/04_h_mask.tif"
+ ],
+ [
+ "images/05_h.jpg",
+ "manual1/05_h.tif",
+ "mask/05_h_mask.tif"
+ ]
+ ],
+ "test": [
+ [
+ "images/06_dr.JPG",
+ "manual1/06_dr.tif",
+ "mask/06_dr_mask.tif"
+ ],
+ [
+ "images/07_dr.JPG",
+ "manual1/07_dr.tif",
+ "mask/07_dr_mask.tif"
+ ],
+ [
+ "images/08_dr.JPG",
+ "manual1/08_dr.tif",
+ "mask/08_dr_mask.tif"
+ ],
+ [
+ "images/09_dr.JPG",
+ "manual1/09_dr.tif",
+ "mask/09_dr_mask.tif"
+ ],
+ [
+ "images/10_dr.JPG",
+ "manual1/10_dr.tif",
+ "mask/10_dr_mask.tif"
+ ],
+ [
+ "images/11_dr.JPG",
+ "manual1/11_dr.tif",
+ "mask/11_dr_mask.tif"
+ ],
+ [
+ "images/12_dr.JPG",
+ "manual1/12_dr.tif",
+ "mask/12_dr_mask.tif"
+ ],
+ [
+ "images/13_dr.JPG",
+ "manual1/13_dr.tif",
+ "mask/13_dr_mask.tif"
+ ],
+ [
+ "images/14_dr.JPG",
+ "manual1/14_dr.tif",
+ "mask/14_dr_mask.tif"
+ ],
+ [
+ "images/15_dr.JPG",
+ "manual1/15_dr.tif",
+ "mask/15_dr_mask.tif"
+ ],
+ [
+ "images/06_g.jpg",
+ "manual1/06_g.tif",
+ "mask/06_g_mask.tif"
+ ],
+ [
+ "images/07_g.jpg",
+ "manual1/07_g.tif",
+ "mask/07_g_mask.tif"
+ ],
+ [
+ "images/08_g.jpg",
+ "manual1/08_g.tif",
+ "mask/08_g_mask.tif"
+ ],
+ [
+ "images/09_g.jpg",
+ "manual1/09_g.tif",
+ "mask/09_g_mask.tif"
+ ],
+ [
+ "images/10_g.jpg",
+ "manual1/10_g.tif",
+ "mask/10_g_mask.tif"
+ ],
+ [
+ "images/11_g.jpg",
+ "manual1/11_g.tif",
+ "mask/11_g_mask.tif"
+ ],
+ [
+ "images/12_g.jpg",
+ "manual1/12_g.tif",
+ "mask/12_g_mask.tif"
+ ],
+ [
+ "images/13_g.jpg",
+ "manual1/13_g.tif",
+ "mask/13_g_mask.tif"
+ ],
+ [
+ "images/14_g.jpg",
+ "manual1/14_g.tif",
+ "mask/14_g_mask.tif"
+ ],
+ [
+ "images/15_g.jpg",
+ "manual1/15_g.tif",
+ "mask/15_g_mask.tif"
+ ],
+ [
+ "images/06_h.jpg",
+ "manual1/06_h.tif",
+ "mask/06_h_mask.tif"
+ ],
+ [
+ "images/07_h.jpg",
+ "manual1/07_h.tif",
+ "mask/07_h_mask.tif"
+ ],
+ [
+ "images/08_h.jpg",
+ "manual1/08_h.tif",
+ "mask/08_h_mask.tif"
+ ],
+ [
+ "images/09_h.jpg",
+ "manual1/09_h.tif",
+ "mask/09_h_mask.tif"
+ ],
+ [
+ "images/10_h.jpg",
+ "manual1/10_h.tif",
+ "mask/10_h_mask.tif"
+ ],
+ [
+ "images/11_h.jpg",
+ "manual1/11_h.tif",
+ "mask/11_h_mask.tif"
+ ],
+ [
+ "images/12_h.jpg",
+ "manual1/12_h.tif",
+ "mask/12_h_mask.tif"
+ ],
+ [
+ "images/13_h.jpg",
+ "manual1/13_h.tif",
+ "mask/13_h_mask.tif"
+ ],
+ [
+ "images/14_h.jpg",
+ "manual1/14_h.tif",
+ "mask/14_h_mask.tif"
+ ],
+ [
+ "images/15_h.jpg",
+ "manual1/15_h.tif",
+ "mask/15_h_mask.tif"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/imagefolder.py b/bob/ip/binseg/data/imagefolder.py
deleted file mode 100644
index 7ec9dd9dbd558b8a1b405076946dca0919df5990..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/data/imagefolder.py
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from torch.utils.data import Dataset
-from pathlib import Path
-import numpy as np
-from PIL import Image
-import torch
-import torchvision.transforms.functional as VF
-import bob.io.base
-
-def get_file_lists(data_path):
- data_path = Path(data_path)
-
- image_path = data_path.joinpath('images')
- image_file_names = np.array(sorted(list(image_path.glob('*'))))
-
- gt_path = data_path.joinpath('gt')
- gt_file_names = np.array(sorted(list(gt_path.glob('*'))))
- return image_file_names, gt_file_names
-
-class ImageFolder(Dataset):
- """
- Generic ImageFolder dataset, that contains two folders:
-
- * ``images`` (vessel images)
- * ``gt`` (ground-truth labels)
-
-
- Parameters
- ----------
- path : str
- full path to root of dataset
-
- """
- def __init__(self, path, transform = None):
- self.transform = transform
- self.img_file_list, self.gt_file_list = get_file_lists(path)
-
- def __len__(self):
- """
- Returns
- -------
- int
- size of the dataset
- """
- return len(self.img_file_list)
-
- def __getitem__(self,index):
- """
- Parameters
- ----------
- index : int
-
- Returns
- -------
- list
- dataitem [img_name, img, gt, mask]
- """
- img_path = self.img_file_list[index]
- img_name = img_path.name
- img = Image.open(img_path).convert(mode='RGB')
-
- gt_path = self.gt_file_list[index]
- if gt_path.suffix == '.hdf5':
- gt = bob.io.base.load(str(gt_path)).astype('float32')
- # not elegant but since transforms require PIL images we do this hacky workaround here
- gt = torch.from_numpy(gt)
- gt = VF.to_pil_image(gt).convert(mode='1', dither=None)
- else:
- gt = Image.open(gt_path).convert(mode='1', dither=None)
-
- sample = [img, gt]
-
- if self.transform :
- sample = self.transform(*sample)
-
- sample.insert(0,img_name)
-
- return sample
diff --git a/bob/ip/binseg/data/imagefolderinference.py b/bob/ip/binseg/data/imagefolderinference.py
deleted file mode 100644
index c4218755f099e5455e474b67fcaa004f99f46ca4..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/data/imagefolderinference.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from torch.utils.data import Dataset
-from pathlib import Path
-import numpy as np
-from PIL import Image
-import torch
-import torchvision.transforms.functional as VF
-import bob.io.base
-
-def get_file_lists(data_path, glob):
- """
- Recursively retrieves file lists from a given path, matching a given glob
-
- This function will use :py:meth:`pathlib.Path.rglob`, together with the
- provided glob pattern to search for anything the desired filename.
- """
-
- data_path = Path(data_path)
- image_file_names = np.array(sorted(list(data_path.rglob(glob))))
- return image_file_names
-
-class ImageFolderInference(Dataset):
- """
- Generic ImageFolder containing images for inference
-
- Notice that this implementation, contrary to its sister
- :py:class:`.ImageFolder`, does not *automatically*
- convert the input image to RGB, before passing it to the transforms, so it
- is possible to accomodate a wider range of input types (e.g. 16-bit PNG
- images).
-
- Parameters
- ----------
- path : str
- full path to root of dataset
-
- glob : str
- glob that can be used to filter-down files to be loaded on the provided
- path
-
- transform : list
- List of transformations to apply to every input sample
-
- """
- def __init__(self, path, glob='*', transform = None):
- self.transform = transform
- self.path = path
- self.img_file_list = get_file_lists(path, glob)
-
- def __len__(self):
- """
- Returns
- -------
- int
- size of the dataset
- """
- return len(self.img_file_list)
-
- def __getitem__(self,index):
- """
- Parameters
- ----------
- index : int
-
- Returns
- -------
- list
- dataitem [img_name, img]
- """
- img_path = self.img_file_list[index]
- img_name = img_path.relative_to(self.path).as_posix()
- img = Image.open(img_path)
-
- sample = [img]
-
- if self.transform :
- sample = self.transform(*sample)
-
- sample.insert(0,img_name)
-
- return sample
diff --git a/bob/ip/binseg/data/iostar/__init__.py b/bob/ip/binseg/data/iostar/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5b762d9d714837f25d60062cc08b3c9be2a1e7d
--- /dev/null
+++ b/bob/ip/binseg/data/iostar/__init__.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""IOSTAR (training set) for Vessel and Optic-Disc Segmentation
+
+The IOSTAR vessel segmentation dataset includes 30 images with a resolution of
+1024 × 1024 pixels. All the vessels in this dataset are annotated by a group of
+experts working in the field of retinal image analysis. Additionally the
+dataset includes annotations for the optic disc and the artery/vein ratio.
+
+* Reference: [IOSTAR-2016]_
+* Original resolution (height x width): 1024 x 1024
+* Split reference: [MEYER-2017]_
+* Protocol ``vessel``:
+
+ * Training samples: 20 (including labels and masks)
+ * Test samples: 10 (including labels and masks)
+
+* Protocol ``optic-disc``:
+
+ * Training samples: 20 (including labels and masks)
+ * Test samples: 10 (including labels and masks)
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+from ..utils import invert_mode1_image, subtract_mode1_images
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "vessel.json"),
+ pkg_resources.resource_filename(__name__, "optic-disc.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.iostar.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _vessel_loader(sample):
+ return dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_1(os.path.join(_root_path, sample["label"])),
+ mask=load_pil_1(os.path.join(_root_path, sample["mask"])),
+ )
+
+
+def _disc_loader(sample):
+ # For optic-disc analysis, the label provided by IOSTAR raw data is the
+ # "inverted" (negative) label, and does not consider the mask region, which
+ # must be subtracted. We do this special manipulation here.
+ data = load_pil_rgb(os.path.join(_root_path, sample["data"]))
+ label = load_pil_1(os.path.join(_root_path, sample["label"]))
+ mask = load_pil_1(os.path.join(_root_path, sample["mask"]))
+ label = subtract_mode1_images(
+ invert_mode1_image(label), invert_mode1_image(mask)
+ )
+ return dict(data=data, label=label, mask=mask)
+
+
+def _loader(context, sample):
+ if context["protocol"] == "optic-disc":
+ return make_delayed(sample, _disc_loader)
+ elif context["protocol"] == "vessel":
+ return make_delayed(sample, _vessel_loader)
+ raise RuntimeError(f"Unknown protocol {context['protocol']}")
+
+
+dataset = JSONDataset(
+ protocols=_protocols, fieldnames=("data", "label", "mask"), loader=_loader,
+)
+"""IOSTAR dataset object"""
diff --git a/bob/ip/binseg/data/iostar/optic-disc.json b/bob/ip/binseg/data/iostar/optic-disc.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f583b6d8d9fbdde805a4a359857a89a428d84c7
--- /dev/null
+++ b/bob/ip/binseg/data/iostar/optic-disc.json
@@ -0,0 +1,156 @@
+{
+ "train": [
+ [
+ "image/STAR 01_OSC.jpg",
+ "mask_OD/STAR 01_OSC_ODMask.tif",
+ "mask/STAR 01_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 02_ODC.jpg",
+ "mask_OD/STAR 02_ODC_ODMask.tif",
+ "mask/STAR 02_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 03_OSN.jpg",
+ "mask_OD/STAR 03_OSN_ODMask.tif",
+ "mask/STAR 03_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 05_ODC.jpg",
+ "mask_OD/STAR 05_ODC_ODMask.tif",
+ "mask/STAR 05_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 06_ODN.jpg",
+ "mask_OD/STAR 06_ODN_ODMask.tif",
+ "mask/STAR 06_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 08_OSN.jpg",
+ "mask_OD/STAR 08_OSN_ODMask.tif",
+ "mask/STAR 08_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 09_OSN.jpg",
+ "mask_OD/STAR 09_OSN_ODMask.tif",
+ "mask/STAR 09_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 10_OSN.jpg",
+ "mask_OD/STAR 10_OSN_ODMask.tif",
+ "mask/STAR 10_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 13_OSN.jpg",
+ "mask_OD/STAR 13_OSN_ODMask.tif",
+ "mask/STAR 13_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 15_OSN.jpg",
+ "mask_OD/STAR 15_OSN_ODMask.tif",
+ "mask/STAR 15_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 16_OSN.jpg",
+ "mask_OD/STAR 16_OSN_ODMask.tif",
+ "mask/STAR 16_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 17_ODN.jpg",
+ "mask_OD/STAR 17_ODN_ODMask.tif",
+ "mask/STAR 17_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 20_ODC.jpg",
+ "mask_OD/STAR 20_ODC_ODMask.tif",
+ "mask/STAR 20_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 21_OSC.jpg",
+ "mask_OD/STAR 21_OSC_ODMask.tif",
+ "mask/STAR 21_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 24_OSC.jpg",
+ "mask_OD/STAR 24_OSC_ODMask.tif",
+ "mask/STAR 24_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 26_ODC.jpg",
+ "mask_OD/STAR 26_ODC_ODMask.tif",
+ "mask/STAR 26_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 28_ODN.jpg",
+ "mask_OD/STAR 28_ODN_ODMask.tif",
+ "mask/STAR 28_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 30_ODC.jpg",
+ "mask_OD/STAR 30_ODC_ODMask.tif",
+ "mask/STAR 30_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 31_ODN.jpg",
+ "mask_OD/STAR 31_ODN_ODMask.tif",
+ "mask/STAR 31_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 32_ODC.jpg",
+ "mask_OD/STAR 32_ODC_ODMask.tif",
+ "mask/STAR 32_ODC_Mask.tif"
+ ]
+ ],
+ "test": [
+ [
+ "image/STAR 34_ODC.jpg",
+ "mask_OD/STAR 34_ODC_ODMask.tif",
+ "mask/STAR 34_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 36_OSC.jpg",
+ "mask_OD/STAR 36_OSC_ODMask.tif",
+ "mask/STAR 36_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 37_ODN.jpg",
+ "mask_OD/STAR 37_ODN_ODMask.tif",
+ "mask/STAR 37_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 38_ODC.jpg",
+ "mask_OD/STAR 38_ODC_ODMask.tif",
+ "mask/STAR 38_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 39_ODC.jpg",
+ "mask_OD/STAR 39_ODC_ODMask.tif",
+ "mask/STAR 39_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 40_OSC.jpg",
+ "mask_OD/STAR 40_OSC_ODMask.tif",
+ "mask/STAR 40_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 43_OSC.jpg",
+ "mask_OD/STAR 43_OSC_ODMask.tif",
+ "mask/STAR 43_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 44_OSN.jpg",
+ "mask_OD/STAR 44_OSN_ODMask.tif",
+ "mask/STAR 44_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 45_ODC.jpg",
+ "mask_OD/STAR 45_ODC_ODMask.tif",
+ "mask/STAR 45_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 48_OSN.jpg",
+ "mask_OD/STAR 48_OSN_ODMask.tif",
+ "mask/STAR 48_OSN_Mask.tif"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/iostar/vessel.json b/bob/ip/binseg/data/iostar/vessel.json
new file mode 100644
index 0000000000000000000000000000000000000000..7500058bc3cf5229da3c6148a2738a66b8e29e8f
--- /dev/null
+++ b/bob/ip/binseg/data/iostar/vessel.json
@@ -0,0 +1,156 @@
+{
+ "train": [
+ [
+ "image/STAR 01_OSC.jpg",
+ "GT/STAR 01_OSC_GT.tif",
+ "mask/STAR 01_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 02_ODC.jpg",
+ "GT/STAR 02_ODC_GT.tif",
+ "mask/STAR 02_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 03_OSN.jpg",
+ "GT/STAR 03_OSN_GT.tif",
+ "mask/STAR 03_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 05_ODC.jpg",
+ "GT/STAR 05_ODC_GT.tif",
+ "mask/STAR 05_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 06_ODN.jpg",
+ "GT/STAR 06_ODN_GT.tif",
+ "mask/STAR 06_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 08_OSN.jpg",
+ "GT/STAR 08_OSN_GT.tif",
+ "mask/STAR 08_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 09_OSN.jpg",
+ "GT/STAR 09_OSN_GT.tif",
+ "mask/STAR 09_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 10_OSN.jpg",
+ "GT/STAR 10_OSN_GT.tif",
+ "mask/STAR 10_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 13_OSN.jpg",
+ "GT/STAR 13_OSN_GT.tif",
+ "mask/STAR 13_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 15_OSN.jpg",
+ "GT/STAR 15_OSN_GT.tif",
+ "mask/STAR 15_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 16_OSN.jpg",
+ "GT/STAR 16_OSN_GT.tif",
+ "mask/STAR 16_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 17_ODN.jpg",
+ "GT/STAR 17_ODN_GT.tif",
+ "mask/STAR 17_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 20_ODC.jpg",
+ "GT/STAR 20_ODC_GT.tif",
+ "mask/STAR 20_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 21_OSC.jpg",
+ "GT/STAR 21_OSC_GT.tif",
+ "mask/STAR 21_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 24_OSC.jpg",
+ "GT/STAR 24_OSC_GT.tif",
+ "mask/STAR 24_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 26_ODC.jpg",
+ "GT/STAR 26_ODC_GT.tif",
+ "mask/STAR 26_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 28_ODN.jpg",
+ "GT/STAR 28_ODN_GT.tif",
+ "mask/STAR 28_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 30_ODC.jpg",
+ "GT/STAR 30_ODC_GT.tif",
+ "mask/STAR 30_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 31_ODN.jpg",
+ "GT/STAR 31_ODN_GT.tif",
+ "mask/STAR 31_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 32_ODC.jpg",
+ "GT/STAR 32_ODC_GT.tif",
+ "mask/STAR 32_ODC_Mask.tif"
+ ]
+ ],
+ "test": [
+ [
+ "image/STAR 34_ODC.jpg",
+ "GT/STAR 34_ODC_GT.tif",
+ "mask/STAR 34_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 36_OSC.jpg",
+ "GT/STAR 36_OSC_GT.tif",
+ "mask/STAR 36_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 37_ODN.jpg",
+ "GT/STAR 37_ODN_GT.tif",
+ "mask/STAR 37_ODN_Mask.tif"
+ ],
+ [
+ "image/STAR 38_ODC.jpg",
+ "GT/STAR 38_ODC_GT.tif",
+ "mask/STAR 38_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 39_ODC.jpg",
+ "GT/STAR 39_ODC_GT.tif",
+ "mask/STAR 39_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 40_OSC.jpg",
+ "GT/STAR 40_OSC_GT.tif",
+ "mask/STAR 40_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 43_OSC.jpg",
+ "GT/STAR 43_OSC_GT.tif",
+ "mask/STAR 43_OSC_Mask.tif"
+ ],
+ [
+ "image/STAR 44_OSN.jpg",
+ "GT/STAR 44_OSN_GT.tif",
+ "mask/STAR 44_OSN_Mask.tif"
+ ],
+ [
+ "image/STAR 45_ODC.jpg",
+ "GT/STAR 45_ODC_GT.tif",
+ "mask/STAR 45_ODC_Mask.tif"
+ ],
+ [
+ "image/STAR 48_OSN.jpg",
+ "GT/STAR 48_OSN_GT.tif",
+ "mask/STAR 48_OSN_Mask.tif"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/loader.py b/bob/ip/binseg/data/loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa2adc416065fb65e25a4b10c5b364758442e8b7
--- /dev/null
+++ b/bob/ip/binseg/data/loader.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Data loading code"""
+
+
+import os
+import functools
+
+import PIL.Image
+
+from .sample import DelayedSample
+
+
+def load_pil_rgb(path):
+ """Loads a sample data
+
+ Parameters
+ ----------
+
+ path : str
+ The full path leading to the image to be loaded
+
+
+ Returns
+ -------
+
+ image : PIL.Image.Image
+ A PIL image in RGB mode
+
+ """
+
+ return PIL.Image.open(path).convert("RGB")
+
+
+def load_pil_1(path):
+ """Loads a sample binary label or mask
+
+ Parameters
+ ----------
+
+ path : str
+ The full path leading to the image to be loaded
+
+
+ Returns
+ -------
+
+ image : PIL.Image.Image
+ A PIL image in mode "1"
+
+ """
+
+ return PIL.Image.open(path).convert(mode="1", dither=None)
+
+
+def make_delayed(sample, loader, key=None):
+ """Returns a delayed-loading Sample object
+
+ Parameters
+ ----------
+
+ sample : dict
+ A dictionary that maps field names to sample data values (e.g. paths)
+
+ loader : object
+ A function that inputs ``sample`` dictionaries and returns the loaded
+ data.
+
+ key : str
+ A unique key identifier for this sample. If not provided, assumes
+ ``sample`` is a dictionary with a ``data`` entry and uses its path as
+ key.
+
+
+ Returns
+ -------
+
+ sample : bob.ip.binseg.data.sample.DelayedSample
+ In which ``key`` is as provided and ``data`` can be accessed to trigger
+ sample loading.
+
+ """
+
+ return DelayedSample(
+ functools.partial(loader, sample),
+ key=key or os.path.splitext(sample["data"])[0],
+ )
diff --git a/bob/ip/binseg/data/refuge/__init__.py b/bob/ip/binseg/data/refuge/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..addcfca9f368bafb68efd0a97a4232f88a5a9d8e
--- /dev/null
+++ b/bob/ip/binseg/data/refuge/__init__.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""REFUGE for Optic Disc and Cup Segmentation
+
+The dataset consists of 1200 color fundus photographs, created for a MICCAI
+challenge. The goal of the challenge is to evaluate and compare automated
+algorithms for glaucoma detection and optic disc/cup segmentation on a common
+dataset of retinal fundus images.
+
+* Reference (including train/dev/test split): [REFUGE-2018]_
+* Protocols ``optic-disc`` and ``cup``:
+
+ * Training samples:
+
+ * 400
+ * includes optic-disc and cup labels
+ * includes label: glaucomatous and non-glaucomatous
+ * original resolution: 2056 x 2124
+
+ * Validation samples:
+
+ * 400
+ * includes optic-disc and cup labels
+ * original resolution: 1634 x 1634
+
+ * Test samples:
+
+ * 400
+ * includes optic-disc and cup labels
+ * includes label: glaucomatous and non-glaucomatous
+ * original resolution:
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, make_delayed
+
+_protocols = {
+ "optic-disc": pkg_resources.resource_filename(__name__, "default.json"),
+ "optic-cup": pkg_resources.resource_filename(__name__, "default.json"),
+}
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.refuge.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _disc_loader(sample):
+ retval = dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_rgb(os.path.join(_root_path, sample["label"])),
+ glaucoma=sample["glaucoma"],
+ )
+ retval["label"] = retval["label"].convert("L")
+ retval["label"] = retval["label"].point(lambda p: p <= 150, mode="1")
+ return retval
+
+
+def _cup_loader(sample):
+ retval = dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])),
+ label=load_pil_rgb(os.path.join(_root_path, sample["label"])),
+ glaucoma=sample["glaucoma"],
+ )
+ retval["label"] = retval["label"].convert("L")
+ retval["label"] = retval["label"].point(lambda p: p <= 100, mode="1")
+ return retval
+
+
+def _loader(context, sample):
+
+ sample["glaucoma"] = False
+ if context["subset"] == "train":
+ # adds binary metadata for glaucoma/non-glaucoma patients
+ sample["glaucoma"] = os.path.basename(sample["label"]).startswith("g")
+ elif context["subset"] == "test":
+ sample["glaucoma"] = (sample["label"].split(os.sep)[-2] == "G")
+ elif context["subset"] == "validation":
+ pass
+ else:
+ raise RuntimeError(f"Unknown subset {context['subset']}")
+
+ # optic disc is drawn with gray == 128 and includes the cup, drawn with
+ # black == 0. The rest is white == 255.
+ if context["protocol"] == "optic-disc":
+ return make_delayed(sample, _disc_loader)
+ elif context["protocol"] == "optic-cup":
+ return make_delayed(sample, _cup_loader)
+ else:
+ raise RuntimeError(f"Unknown protocol {context['protocol']}")
+
+
+dataset = JSONDataset(
+ protocols=_protocols,
+ fieldnames=("data", "label"),
+ loader=_loader,
+)
+"""REFUGE dataset object"""
diff --git a/bob/ip/binseg/data/refuge/default.json b/bob/ip/binseg/data/refuge/default.json
new file mode 100644
index 0000000000000000000000000000000000000000..cde130a04f5c7240698cdf3382142f0ccacb527e
--- /dev/null
+++ b/bob/ip/binseg/data/refuge/default.json
@@ -0,0 +1,4808 @@
+{
+ "train": [
+ [
+ "Training400/Glaucoma/g0001.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0001.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0002.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0002.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0003.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0003.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0004.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0004.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0005.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0005.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0006.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0006.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0007.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0007.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0008.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0008.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0009.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0009.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0010.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0010.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0011.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0011.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0012.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0012.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0013.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0013.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0014.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0014.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0015.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0015.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0016.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0016.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0017.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0017.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0018.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0018.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0019.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0019.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0020.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0020.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0021.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0021.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0022.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0022.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0023.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0023.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0024.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0024.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0025.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0025.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0026.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0026.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0027.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0027.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0028.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0028.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0029.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0029.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0030.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0030.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0031.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0031.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0032.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0032.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0033.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0033.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0034.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0034.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0035.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0035.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0036.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0036.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0037.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0037.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0038.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0038.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0039.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0039.bmp"
+ ],
+ [
+ "Training400/Glaucoma/g0040.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Glaucoma/g0040.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0001.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0001.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0002.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0002.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0003.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0003.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0004.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0004.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0005.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0005.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0006.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0006.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0007.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0007.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0008.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0008.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0009.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0009.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0010.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0010.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0011.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0011.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0012.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0012.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0013.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0013.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0014.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0014.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0015.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0015.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0016.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0016.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0017.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0017.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0018.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0018.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0019.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0019.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0020.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0020.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0021.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0021.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0022.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0022.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0023.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0023.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0024.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0024.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0025.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0025.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0026.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0026.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0027.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0027.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0028.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0028.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0029.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0029.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0030.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0030.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0031.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0031.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0032.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0032.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0033.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0033.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0034.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0034.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0035.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0035.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0036.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0036.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0037.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0037.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0038.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0038.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0039.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0039.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0040.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0040.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0041.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0041.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0042.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0042.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0043.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0043.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0044.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0044.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0045.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0045.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0046.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0046.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0047.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0047.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0048.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0048.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0049.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0049.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0050.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0050.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0051.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0051.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0052.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0052.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0053.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0053.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0054.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0054.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0055.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0055.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0056.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0056.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0057.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0057.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0058.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0058.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0059.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0059.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0060.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0060.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0061.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0061.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0062.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0062.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0063.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0063.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0064.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0064.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0065.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0065.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0066.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0066.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0067.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0067.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0068.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0068.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0069.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0069.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0070.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0070.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0071.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0071.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0072.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0072.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0073.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0073.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0074.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0074.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0075.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0075.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0076.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0076.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0077.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0077.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0078.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0078.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0079.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0079.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0080.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0080.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0081.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0081.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0082.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0082.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0083.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0083.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0084.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0084.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0085.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0085.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0086.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0086.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0087.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0087.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0088.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0088.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0089.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0089.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0090.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0090.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0091.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0091.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0092.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0092.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0093.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0093.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0094.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0094.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0095.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0095.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0096.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0096.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0097.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0097.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0098.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0098.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0099.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0099.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0100.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0100.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0101.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0101.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0102.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0102.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0103.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0103.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0104.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0104.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0105.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0105.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0106.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0106.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0107.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0107.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0108.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0108.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0109.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0109.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0110.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0110.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0111.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0111.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0112.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0112.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0113.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0113.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0114.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0114.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0115.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0115.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0116.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0116.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0117.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0117.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0118.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0118.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0119.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0119.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0120.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0120.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0121.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0121.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0122.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0122.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0123.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0123.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0124.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0124.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0125.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0125.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0126.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0126.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0127.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0127.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0128.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0128.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0129.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0129.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0130.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0130.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0131.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0131.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0132.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0132.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0133.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0133.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0134.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0134.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0135.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0135.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0136.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0136.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0137.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0137.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0138.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0138.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0139.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0139.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0140.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0140.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0141.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0141.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0142.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0142.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0143.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0143.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0144.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0144.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0145.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0145.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0146.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0146.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0147.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0147.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0148.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0148.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0149.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0149.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0150.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0150.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0151.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0151.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0152.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0152.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0153.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0153.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0154.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0154.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0155.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0155.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0156.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0156.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0157.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0157.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0158.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0158.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0159.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0159.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0160.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0160.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0161.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0161.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0162.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0162.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0163.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0163.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0164.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0164.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0165.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0165.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0166.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0166.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0167.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0167.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0168.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0168.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0169.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0169.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0170.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0170.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0171.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0171.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0172.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0172.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0173.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0173.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0174.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0174.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0175.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0175.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0176.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0176.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0177.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0177.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0178.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0178.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0179.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0179.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0180.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0180.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0181.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0181.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0182.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0182.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0183.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0183.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0184.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0184.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0185.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0185.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0186.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0186.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0187.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0187.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0188.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0188.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0189.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0189.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0190.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0190.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0191.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0191.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0192.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0192.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0193.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0193.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0194.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0194.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0195.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0195.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0196.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0196.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0197.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0197.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0198.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0198.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0199.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0199.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0200.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0200.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0201.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0201.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0202.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0202.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0203.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0203.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0204.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0204.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0205.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0205.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0206.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0206.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0207.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0207.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0208.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0208.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0209.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0209.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0210.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0210.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0211.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0211.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0212.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0212.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0213.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0213.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0214.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0214.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0215.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0215.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0216.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0216.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0217.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0217.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0218.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0218.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0219.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0219.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0220.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0220.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0221.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0221.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0222.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0222.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0223.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0223.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0224.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0224.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0225.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0225.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0226.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0226.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0227.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0227.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0228.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0228.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0229.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0229.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0230.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0230.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0231.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0231.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0232.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0232.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0233.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0233.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0234.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0234.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0235.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0235.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0236.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0236.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0237.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0237.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0238.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0238.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0239.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0239.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0240.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0240.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0241.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0241.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0242.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0242.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0243.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0243.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0244.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0244.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0245.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0245.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0246.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0246.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0247.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0247.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0248.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0248.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0249.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0249.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0250.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0250.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0251.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0251.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0252.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0252.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0253.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0253.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0254.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0254.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0255.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0255.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0256.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0256.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0257.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0257.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0258.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0258.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0259.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0259.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0260.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0260.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0261.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0261.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0262.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0262.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0263.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0263.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0264.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0264.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0265.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0265.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0266.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0266.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0267.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0267.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0268.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0268.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0269.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0269.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0270.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0270.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0271.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0271.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0272.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0272.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0273.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0273.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0274.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0274.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0275.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0275.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0276.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0276.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0277.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0277.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0278.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0278.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0279.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0279.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0280.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0280.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0281.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0281.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0282.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0282.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0283.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0283.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0284.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0284.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0285.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0285.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0286.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0286.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0287.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0287.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0288.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0288.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0289.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0289.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0290.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0290.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0291.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0291.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0292.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0292.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0293.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0293.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0294.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0294.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0295.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0295.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0296.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0296.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0297.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0297.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0298.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0298.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0299.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0299.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0300.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0300.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0301.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0301.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0302.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0302.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0303.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0303.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0304.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0304.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0305.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0305.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0306.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0306.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0307.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0307.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0308.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0308.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0309.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0309.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0310.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0310.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0311.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0311.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0312.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0312.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0313.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0313.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0314.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0314.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0315.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0315.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0316.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0316.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0317.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0317.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0318.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0318.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0319.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0319.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0320.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0320.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0321.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0321.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0322.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0322.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0323.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0323.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0324.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0324.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0325.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0325.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0326.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0326.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0327.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0327.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0328.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0328.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0329.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0329.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0330.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0330.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0331.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0331.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0332.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0332.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0333.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0333.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0334.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0334.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0335.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0335.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0336.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0336.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0337.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0337.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0338.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0338.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0339.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0339.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0340.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0340.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0341.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0341.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0342.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0342.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0343.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0343.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0344.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0344.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0345.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0345.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0346.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0346.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0347.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0347.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0348.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0348.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0349.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0349.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0350.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0350.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0351.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0351.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0352.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0352.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0353.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0353.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0354.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0354.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0355.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0355.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0356.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0356.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0357.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0357.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0358.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0358.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0359.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0359.bmp"
+ ],
+ [
+ "Training400/Non-Glaucoma/n0360.jpg",
+ "Annotation-Training400/Disc_Cup_Masks/Non-Glaucoma/n0360.bmp"
+ ]
+ ],
+ "validation": [
+ [
+ "REFUGE-Validation400/V0001.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0001.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0002.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0002.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0003.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0003.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0004.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0004.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0005.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0005.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0006.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0006.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0007.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0007.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0008.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0008.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0009.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0009.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0010.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0010.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0011.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0011.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0012.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0012.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0013.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0013.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0014.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0014.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0015.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0015.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0016.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0016.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0017.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0017.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0018.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0018.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0019.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0019.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0020.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0020.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0021.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0021.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0022.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0022.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0023.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0023.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0024.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0024.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0025.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0025.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0026.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0026.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0027.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0027.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0028.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0028.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0029.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0029.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0030.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0030.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0031.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0031.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0032.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0032.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0033.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0033.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0034.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0034.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0035.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0035.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0036.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0036.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0037.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0037.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0038.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0038.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0039.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0039.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0040.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0040.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0041.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0041.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0042.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0042.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0043.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0043.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0044.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0044.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0045.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0045.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0046.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0046.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0047.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0047.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0048.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0048.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0049.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0049.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0050.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0050.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0051.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0051.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0052.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0052.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0053.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0053.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0054.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0054.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0055.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0055.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0056.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0056.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0057.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0057.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0058.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0058.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0059.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0059.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0060.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0060.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0061.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0061.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0062.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0062.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0063.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0063.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0064.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0064.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0065.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0065.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0066.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0066.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0067.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0067.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0068.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0068.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0069.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0069.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0070.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0070.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0071.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0071.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0072.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0072.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0073.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0073.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0074.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0074.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0075.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0075.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0076.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0076.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0077.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0077.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0078.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0078.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0079.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0079.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0080.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0080.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0081.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0081.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0082.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0082.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0083.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0083.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0084.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0084.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0085.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0085.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0086.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0086.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0087.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0087.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0088.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0088.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0089.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0089.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0090.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0090.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0091.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0091.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0092.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0092.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0093.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0093.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0094.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0094.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0095.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0095.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0096.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0096.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0097.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0097.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0098.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0098.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0099.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0099.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0100.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0100.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0101.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0101.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0102.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0102.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0103.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0103.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0104.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0104.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0105.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0105.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0106.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0106.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0107.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0107.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0108.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0108.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0109.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0109.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0110.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0110.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0111.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0111.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0112.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0112.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0113.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0113.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0114.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0114.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0115.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0115.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0116.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0116.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0117.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0117.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0118.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0118.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0119.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0119.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0120.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0120.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0121.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0121.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0122.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0122.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0123.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0123.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0124.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0124.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0125.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0125.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0126.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0126.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0127.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0127.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0128.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0128.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0129.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0129.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0130.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0130.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0131.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0131.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0132.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0132.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0133.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0133.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0134.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0134.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0135.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0135.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0136.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0136.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0137.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0137.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0138.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0138.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0139.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0139.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0140.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0140.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0141.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0141.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0142.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0142.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0143.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0143.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0144.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0144.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0145.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0145.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0146.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0146.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0147.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0147.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0148.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0148.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0149.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0149.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0150.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0150.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0151.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0151.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0152.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0152.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0153.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0153.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0154.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0154.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0155.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0155.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0156.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0156.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0157.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0157.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0158.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0158.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0159.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0159.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0160.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0160.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0161.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0161.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0162.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0162.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0163.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0163.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0164.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0164.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0165.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0165.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0166.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0166.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0167.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0167.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0168.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0168.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0169.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0169.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0170.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0170.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0171.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0171.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0172.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0172.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0173.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0173.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0174.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0174.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0175.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0175.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0176.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0176.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0177.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0177.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0178.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0178.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0179.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0179.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0180.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0180.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0181.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0181.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0182.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0182.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0183.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0183.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0184.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0184.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0185.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0185.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0186.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0186.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0187.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0187.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0188.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0188.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0189.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0189.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0190.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0190.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0191.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0191.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0192.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0192.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0193.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0193.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0194.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0194.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0195.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0195.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0196.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0196.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0197.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0197.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0198.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0198.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0199.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0199.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0200.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0200.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0201.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0201.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0202.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0202.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0203.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0203.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0204.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0204.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0205.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0205.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0206.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0206.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0207.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0207.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0208.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0208.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0209.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0209.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0210.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0210.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0211.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0211.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0212.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0212.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0213.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0213.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0214.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0214.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0215.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0215.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0216.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0216.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0217.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0217.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0218.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0218.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0219.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0219.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0220.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0220.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0221.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0221.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0222.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0222.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0223.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0223.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0224.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0224.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0225.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0225.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0226.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0226.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0227.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0227.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0228.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0228.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0229.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0229.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0230.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0230.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0231.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0231.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0232.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0232.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0233.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0233.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0234.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0234.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0235.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0235.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0236.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0236.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0237.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0237.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0238.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0238.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0239.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0239.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0240.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0240.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0241.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0241.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0242.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0242.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0243.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0243.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0244.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0244.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0245.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0245.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0246.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0246.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0247.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0247.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0248.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0248.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0249.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0249.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0250.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0250.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0251.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0251.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0252.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0252.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0253.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0253.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0254.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0254.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0255.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0255.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0256.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0256.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0257.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0257.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0258.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0258.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0259.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0259.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0260.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0260.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0261.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0261.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0262.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0262.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0263.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0263.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0264.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0264.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0265.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0265.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0266.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0266.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0267.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0267.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0268.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0268.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0269.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0269.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0270.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0270.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0271.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0271.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0272.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0272.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0273.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0273.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0274.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0274.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0275.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0275.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0276.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0276.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0277.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0277.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0278.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0278.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0279.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0279.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0280.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0280.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0281.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0281.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0282.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0282.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0283.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0283.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0284.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0284.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0285.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0285.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0286.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0286.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0287.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0287.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0288.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0288.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0289.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0289.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0290.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0290.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0291.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0291.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0292.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0292.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0293.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0293.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0294.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0294.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0295.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0295.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0296.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0296.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0297.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0297.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0298.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0298.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0299.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0299.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0300.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0300.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0301.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0301.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0302.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0302.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0303.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0303.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0304.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0304.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0305.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0305.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0306.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0306.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0307.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0307.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0308.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0308.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0309.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0309.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0310.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0310.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0311.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0311.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0312.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0312.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0313.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0313.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0314.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0314.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0315.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0315.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0316.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0316.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0317.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0317.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0318.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0318.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0319.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0319.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0320.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0320.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0321.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0321.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0322.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0322.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0323.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0323.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0324.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0324.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0325.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0325.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0326.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0326.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0327.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0327.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0328.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0328.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0329.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0329.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0330.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0330.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0331.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0331.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0332.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0332.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0333.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0333.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0334.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0334.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0335.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0335.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0336.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0336.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0337.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0337.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0338.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0338.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0339.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0339.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0340.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0340.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0341.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0341.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0342.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0342.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0343.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0343.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0344.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0344.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0345.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0345.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0346.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0346.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0347.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0347.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0348.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0348.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0349.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0349.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0350.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0350.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0351.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0351.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0352.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0352.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0353.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0353.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0354.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0354.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0355.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0355.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0356.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0356.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0357.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0357.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0358.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0358.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0359.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0359.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0360.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0360.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0361.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0361.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0362.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0362.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0363.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0363.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0364.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0364.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0365.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0365.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0366.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0366.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0367.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0367.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0368.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0368.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0369.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0369.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0370.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0370.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0371.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0371.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0372.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0372.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0373.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0373.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0374.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0374.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0375.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0375.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0376.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0376.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0377.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0377.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0378.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0378.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0379.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0379.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0380.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0380.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0381.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0381.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0382.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0382.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0383.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0383.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0384.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0384.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0385.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0385.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0386.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0386.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0387.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0387.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0388.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0388.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0389.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0389.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0390.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0390.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0391.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0391.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0392.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0392.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0393.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0393.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0394.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0394.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0395.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0395.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0396.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0396.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0397.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0397.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0398.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0398.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0399.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0399.bmp"
+ ],
+ [
+ "REFUGE-Validation400/V0400.jpg",
+ "REFUGE-Validation400-GT/Disc_Cup_Masks/V0400.bmp"
+ ]
+ ],
+ "test": [
+ [
+ "Test400/T0001.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0001.bmp"
+ ],
+ [
+ "Test400/T0002.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0002.bmp"
+ ],
+ [
+ "Test400/T0003.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0003.bmp"
+ ],
+ [
+ "Test400/T0004.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0004.bmp"
+ ],
+ [
+ "Test400/T0005.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0005.bmp"
+ ],
+ [
+ "Test400/T0006.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0006.bmp"
+ ],
+ [
+ "Test400/T0007.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0007.bmp"
+ ],
+ [
+ "Test400/T0008.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0008.bmp"
+ ],
+ [
+ "Test400/T0009.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0009.bmp"
+ ],
+ [
+ "Test400/T0010.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0010.bmp"
+ ],
+ [
+ "Test400/T0011.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0011.bmp"
+ ],
+ [
+ "Test400/T0012.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0012.bmp"
+ ],
+ [
+ "Test400/T0013.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0013.bmp"
+ ],
+ [
+ "Test400/T0014.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0014.bmp"
+ ],
+ [
+ "Test400/T0015.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0015.bmp"
+ ],
+ [
+ "Test400/T0016.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0016.bmp"
+ ],
+ [
+ "Test400/T0017.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0017.bmp"
+ ],
+ [
+ "Test400/T0018.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0018.bmp"
+ ],
+ [
+ "Test400/T0019.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0019.bmp"
+ ],
+ [
+ "Test400/T0020.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0020.bmp"
+ ],
+ [
+ "Test400/T0021.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0021.bmp"
+ ],
+ [
+ "Test400/T0022.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0022.bmp"
+ ],
+ [
+ "Test400/T0023.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0023.bmp"
+ ],
+ [
+ "Test400/T0024.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0024.bmp"
+ ],
+ [
+ "Test400/T0025.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0025.bmp"
+ ],
+ [
+ "Test400/T0026.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0026.bmp"
+ ],
+ [
+ "Test400/T0027.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0027.bmp"
+ ],
+ [
+ "Test400/T0028.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0028.bmp"
+ ],
+ [
+ "Test400/T0029.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0029.bmp"
+ ],
+ [
+ "Test400/T0030.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0030.bmp"
+ ],
+ [
+ "Test400/T0031.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0031.bmp"
+ ],
+ [
+ "Test400/T0032.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0032.bmp"
+ ],
+ [
+ "Test400/T0033.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0033.bmp"
+ ],
+ [
+ "Test400/T0034.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0034.bmp"
+ ],
+ [
+ "Test400/T0035.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0035.bmp"
+ ],
+ [
+ "Test400/T0036.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0036.bmp"
+ ],
+ [
+ "Test400/T0037.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0037.bmp"
+ ],
+ [
+ "Test400/T0038.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0038.bmp"
+ ],
+ [
+ "Test400/T0039.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0039.bmp"
+ ],
+ [
+ "Test400/T0040.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0040.bmp"
+ ],
+ [
+ "Test400/T0041.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0041.bmp"
+ ],
+ [
+ "Test400/T0042.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0042.bmp"
+ ],
+ [
+ "Test400/T0043.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0043.bmp"
+ ],
+ [
+ "Test400/T0044.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0044.bmp"
+ ],
+ [
+ "Test400/T0045.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0045.bmp"
+ ],
+ [
+ "Test400/T0046.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0046.bmp"
+ ],
+ [
+ "Test400/T0047.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0047.bmp"
+ ],
+ [
+ "Test400/T0048.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0048.bmp"
+ ],
+ [
+ "Test400/T0049.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0049.bmp"
+ ],
+ [
+ "Test400/T0050.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0050.bmp"
+ ],
+ [
+ "Test400/T0051.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0051.bmp"
+ ],
+ [
+ "Test400/T0052.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0052.bmp"
+ ],
+ [
+ "Test400/T0053.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0053.bmp"
+ ],
+ [
+ "Test400/T0054.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0054.bmp"
+ ],
+ [
+ "Test400/T0055.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0055.bmp"
+ ],
+ [
+ "Test400/T0056.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0056.bmp"
+ ],
+ [
+ "Test400/T0057.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0057.bmp"
+ ],
+ [
+ "Test400/T0058.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0058.bmp"
+ ],
+ [
+ "Test400/T0059.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0059.bmp"
+ ],
+ [
+ "Test400/T0060.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0060.bmp"
+ ],
+ [
+ "Test400/T0061.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0061.bmp"
+ ],
+ [
+ "Test400/T0062.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0062.bmp"
+ ],
+ [
+ "Test400/T0063.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0063.bmp"
+ ],
+ [
+ "Test400/T0064.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0064.bmp"
+ ],
+ [
+ "Test400/T0065.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0065.bmp"
+ ],
+ [
+ "Test400/T0066.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0066.bmp"
+ ],
+ [
+ "Test400/T0067.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0067.bmp"
+ ],
+ [
+ "Test400/T0068.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0068.bmp"
+ ],
+ [
+ "Test400/T0069.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0069.bmp"
+ ],
+ [
+ "Test400/T0070.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0070.bmp"
+ ],
+ [
+ "Test400/T0071.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0071.bmp"
+ ],
+ [
+ "Test400/T0072.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0072.bmp"
+ ],
+ [
+ "Test400/T0073.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0073.bmp"
+ ],
+ [
+ "Test400/T0074.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0074.bmp"
+ ],
+ [
+ "Test400/T0075.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0075.bmp"
+ ],
+ [
+ "Test400/T0076.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0076.bmp"
+ ],
+ [
+ "Test400/T0077.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0077.bmp"
+ ],
+ [
+ "Test400/T0078.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0078.bmp"
+ ],
+ [
+ "Test400/T0079.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0079.bmp"
+ ],
+ [
+ "Test400/T0080.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0080.bmp"
+ ],
+ [
+ "Test400/T0081.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0081.bmp"
+ ],
+ [
+ "Test400/T0082.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0082.bmp"
+ ],
+ [
+ "Test400/T0083.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0083.bmp"
+ ],
+ [
+ "Test400/T0084.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0084.bmp"
+ ],
+ [
+ "Test400/T0085.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0085.bmp"
+ ],
+ [
+ "Test400/T0086.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0086.bmp"
+ ],
+ [
+ "Test400/T0087.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0087.bmp"
+ ],
+ [
+ "Test400/T0088.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0088.bmp"
+ ],
+ [
+ "Test400/T0089.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0089.bmp"
+ ],
+ [
+ "Test400/T0090.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0090.bmp"
+ ],
+ [
+ "Test400/T0091.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0091.bmp"
+ ],
+ [
+ "Test400/T0092.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0092.bmp"
+ ],
+ [
+ "Test400/T0093.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0093.bmp"
+ ],
+ [
+ "Test400/T0094.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0094.bmp"
+ ],
+ [
+ "Test400/T0095.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0095.bmp"
+ ],
+ [
+ "Test400/T0096.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0096.bmp"
+ ],
+ [
+ "Test400/T0097.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0097.bmp"
+ ],
+ [
+ "Test400/T0098.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0098.bmp"
+ ],
+ [
+ "Test400/T0099.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0099.bmp"
+ ],
+ [
+ "Test400/T0100.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0100.bmp"
+ ],
+ [
+ "Test400/T0101.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0101.bmp"
+ ],
+ [
+ "Test400/T0102.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0102.bmp"
+ ],
+ [
+ "Test400/T0103.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0103.bmp"
+ ],
+ [
+ "Test400/T0104.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0104.bmp"
+ ],
+ [
+ "Test400/T0105.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0105.bmp"
+ ],
+ [
+ "Test400/T0106.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0106.bmp"
+ ],
+ [
+ "Test400/T0107.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0107.bmp"
+ ],
+ [
+ "Test400/T0108.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0108.bmp"
+ ],
+ [
+ "Test400/T0109.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0109.bmp"
+ ],
+ [
+ "Test400/T0110.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0110.bmp"
+ ],
+ [
+ "Test400/T0111.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0111.bmp"
+ ],
+ [
+ "Test400/T0112.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0112.bmp"
+ ],
+ [
+ "Test400/T0113.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0113.bmp"
+ ],
+ [
+ "Test400/T0114.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0114.bmp"
+ ],
+ [
+ "Test400/T0115.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0115.bmp"
+ ],
+ [
+ "Test400/T0116.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0116.bmp"
+ ],
+ [
+ "Test400/T0117.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0117.bmp"
+ ],
+ [
+ "Test400/T0118.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0118.bmp"
+ ],
+ [
+ "Test400/T0119.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0119.bmp"
+ ],
+ [
+ "Test400/T0120.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0120.bmp"
+ ],
+ [
+ "Test400/T0121.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0121.bmp"
+ ],
+ [
+ "Test400/T0122.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0122.bmp"
+ ],
+ [
+ "Test400/T0123.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0123.bmp"
+ ],
+ [
+ "Test400/T0124.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0124.bmp"
+ ],
+ [
+ "Test400/T0125.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0125.bmp"
+ ],
+ [
+ "Test400/T0126.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0126.bmp"
+ ],
+ [
+ "Test400/T0127.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0127.bmp"
+ ],
+ [
+ "Test400/T0128.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0128.bmp"
+ ],
+ [
+ "Test400/T0129.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0129.bmp"
+ ],
+ [
+ "Test400/T0130.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0130.bmp"
+ ],
+ [
+ "Test400/T0131.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0131.bmp"
+ ],
+ [
+ "Test400/T0132.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0132.bmp"
+ ],
+ [
+ "Test400/T0133.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0133.bmp"
+ ],
+ [
+ "Test400/T0134.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0134.bmp"
+ ],
+ [
+ "Test400/T0135.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0135.bmp"
+ ],
+ [
+ "Test400/T0136.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0136.bmp"
+ ],
+ [
+ "Test400/T0137.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0137.bmp"
+ ],
+ [
+ "Test400/T0138.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0138.bmp"
+ ],
+ [
+ "Test400/T0139.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0139.bmp"
+ ],
+ [
+ "Test400/T0140.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0140.bmp"
+ ],
+ [
+ "Test400/T0141.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0141.bmp"
+ ],
+ [
+ "Test400/T0142.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0142.bmp"
+ ],
+ [
+ "Test400/T0143.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0143.bmp"
+ ],
+ [
+ "Test400/T0144.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0144.bmp"
+ ],
+ [
+ "Test400/T0145.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0145.bmp"
+ ],
+ [
+ "Test400/T0146.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0146.bmp"
+ ],
+ [
+ "Test400/T0147.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0147.bmp"
+ ],
+ [
+ "Test400/T0148.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0148.bmp"
+ ],
+ [
+ "Test400/T0149.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0149.bmp"
+ ],
+ [
+ "Test400/T0150.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0150.bmp"
+ ],
+ [
+ "Test400/T0151.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0151.bmp"
+ ],
+ [
+ "Test400/T0152.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0152.bmp"
+ ],
+ [
+ "Test400/T0153.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0153.bmp"
+ ],
+ [
+ "Test400/T0154.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0154.bmp"
+ ],
+ [
+ "Test400/T0155.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0155.bmp"
+ ],
+ [
+ "Test400/T0156.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0156.bmp"
+ ],
+ [
+ "Test400/T0157.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0157.bmp"
+ ],
+ [
+ "Test400/T0158.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0158.bmp"
+ ],
+ [
+ "Test400/T0159.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0159.bmp"
+ ],
+ [
+ "Test400/T0160.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0160.bmp"
+ ],
+ [
+ "Test400/T0161.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0161.bmp"
+ ],
+ [
+ "Test400/T0162.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0162.bmp"
+ ],
+ [
+ "Test400/T0163.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0163.bmp"
+ ],
+ [
+ "Test400/T0164.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0164.bmp"
+ ],
+ [
+ "Test400/T0165.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0165.bmp"
+ ],
+ [
+ "Test400/T0166.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0166.bmp"
+ ],
+ [
+ "Test400/T0167.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0167.bmp"
+ ],
+ [
+ "Test400/T0168.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0168.bmp"
+ ],
+ [
+ "Test400/T0169.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0169.bmp"
+ ],
+ [
+ "Test400/T0170.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0170.bmp"
+ ],
+ [
+ "Test400/T0171.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0171.bmp"
+ ],
+ [
+ "Test400/T0172.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0172.bmp"
+ ],
+ [
+ "Test400/T0173.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0173.bmp"
+ ],
+ [
+ "Test400/T0174.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0174.bmp"
+ ],
+ [
+ "Test400/T0175.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0175.bmp"
+ ],
+ [
+ "Test400/T0176.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0176.bmp"
+ ],
+ [
+ "Test400/T0177.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0177.bmp"
+ ],
+ [
+ "Test400/T0178.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0178.bmp"
+ ],
+ [
+ "Test400/T0179.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0179.bmp"
+ ],
+ [
+ "Test400/T0180.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0180.bmp"
+ ],
+ [
+ "Test400/T0181.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0181.bmp"
+ ],
+ [
+ "Test400/T0182.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0182.bmp"
+ ],
+ [
+ "Test400/T0183.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0183.bmp"
+ ],
+ [
+ "Test400/T0184.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0184.bmp"
+ ],
+ [
+ "Test400/T0185.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0185.bmp"
+ ],
+ [
+ "Test400/T0186.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0186.bmp"
+ ],
+ [
+ "Test400/T0187.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0187.bmp"
+ ],
+ [
+ "Test400/T0188.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0188.bmp"
+ ],
+ [
+ "Test400/T0189.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0189.bmp"
+ ],
+ [
+ "Test400/T0190.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0190.bmp"
+ ],
+ [
+ "Test400/T0191.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0191.bmp"
+ ],
+ [
+ "Test400/T0192.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0192.bmp"
+ ],
+ [
+ "Test400/T0193.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0193.bmp"
+ ],
+ [
+ "Test400/T0194.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0194.bmp"
+ ],
+ [
+ "Test400/T0195.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0195.bmp"
+ ],
+ [
+ "Test400/T0196.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0196.bmp"
+ ],
+ [
+ "Test400/T0197.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0197.bmp"
+ ],
+ [
+ "Test400/T0198.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0198.bmp"
+ ],
+ [
+ "Test400/T0199.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0199.bmp"
+ ],
+ [
+ "Test400/T0200.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0200.bmp"
+ ],
+ [
+ "Test400/T0201.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0201.bmp"
+ ],
+ [
+ "Test400/T0202.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0202.bmp"
+ ],
+ [
+ "Test400/T0203.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0203.bmp"
+ ],
+ [
+ "Test400/T0204.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0204.bmp"
+ ],
+ [
+ "Test400/T0205.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0205.bmp"
+ ],
+ [
+ "Test400/T0206.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0206.bmp"
+ ],
+ [
+ "Test400/T0207.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0207.bmp"
+ ],
+ [
+ "Test400/T0208.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0208.bmp"
+ ],
+ [
+ "Test400/T0209.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0209.bmp"
+ ],
+ [
+ "Test400/T0210.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0210.bmp"
+ ],
+ [
+ "Test400/T0211.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0211.bmp"
+ ],
+ [
+ "Test400/T0212.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0212.bmp"
+ ],
+ [
+ "Test400/T0213.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0213.bmp"
+ ],
+ [
+ "Test400/T0214.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0214.bmp"
+ ],
+ [
+ "Test400/T0215.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0215.bmp"
+ ],
+ [
+ "Test400/T0216.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0216.bmp"
+ ],
+ [
+ "Test400/T0217.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0217.bmp"
+ ],
+ [
+ "Test400/T0218.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0218.bmp"
+ ],
+ [
+ "Test400/T0219.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0219.bmp"
+ ],
+ [
+ "Test400/T0220.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0220.bmp"
+ ],
+ [
+ "Test400/T0221.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0221.bmp"
+ ],
+ [
+ "Test400/T0222.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0222.bmp"
+ ],
+ [
+ "Test400/T0223.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0223.bmp"
+ ],
+ [
+ "Test400/T0224.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0224.bmp"
+ ],
+ [
+ "Test400/T0225.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0225.bmp"
+ ],
+ [
+ "Test400/T0226.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0226.bmp"
+ ],
+ [
+ "Test400/T0227.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0227.bmp"
+ ],
+ [
+ "Test400/T0228.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0228.bmp"
+ ],
+ [
+ "Test400/T0229.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0229.bmp"
+ ],
+ [
+ "Test400/T0230.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0230.bmp"
+ ],
+ [
+ "Test400/T0231.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0231.bmp"
+ ],
+ [
+ "Test400/T0232.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0232.bmp"
+ ],
+ [
+ "Test400/T0233.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0233.bmp"
+ ],
+ [
+ "Test400/T0234.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0234.bmp"
+ ],
+ [
+ "Test400/T0235.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0235.bmp"
+ ],
+ [
+ "Test400/T0236.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0236.bmp"
+ ],
+ [
+ "Test400/T0237.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0237.bmp"
+ ],
+ [
+ "Test400/T0238.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0238.bmp"
+ ],
+ [
+ "Test400/T0239.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0239.bmp"
+ ],
+ [
+ "Test400/T0240.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0240.bmp"
+ ],
+ [
+ "Test400/T0241.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0241.bmp"
+ ],
+ [
+ "Test400/T0242.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0242.bmp"
+ ],
+ [
+ "Test400/T0243.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0243.bmp"
+ ],
+ [
+ "Test400/T0244.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0244.bmp"
+ ],
+ [
+ "Test400/T0245.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0245.bmp"
+ ],
+ [
+ "Test400/T0246.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0246.bmp"
+ ],
+ [
+ "Test400/T0247.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0247.bmp"
+ ],
+ [
+ "Test400/T0248.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0248.bmp"
+ ],
+ [
+ "Test400/T0249.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0249.bmp"
+ ],
+ [
+ "Test400/T0250.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0250.bmp"
+ ],
+ [
+ "Test400/T0251.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0251.bmp"
+ ],
+ [
+ "Test400/T0252.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0252.bmp"
+ ],
+ [
+ "Test400/T0253.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0253.bmp"
+ ],
+ [
+ "Test400/T0254.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0254.bmp"
+ ],
+ [
+ "Test400/T0255.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0255.bmp"
+ ],
+ [
+ "Test400/T0256.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0256.bmp"
+ ],
+ [
+ "Test400/T0257.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0257.bmp"
+ ],
+ [
+ "Test400/T0258.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0258.bmp"
+ ],
+ [
+ "Test400/T0259.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0259.bmp"
+ ],
+ [
+ "Test400/T0260.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0260.bmp"
+ ],
+ [
+ "Test400/T0261.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0261.bmp"
+ ],
+ [
+ "Test400/T0262.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0262.bmp"
+ ],
+ [
+ "Test400/T0263.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0263.bmp"
+ ],
+ [
+ "Test400/T0264.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0264.bmp"
+ ],
+ [
+ "Test400/T0265.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0265.bmp"
+ ],
+ [
+ "Test400/T0266.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0266.bmp"
+ ],
+ [
+ "Test400/T0267.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0267.bmp"
+ ],
+ [
+ "Test400/T0268.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0268.bmp"
+ ],
+ [
+ "Test400/T0269.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0269.bmp"
+ ],
+ [
+ "Test400/T0270.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0270.bmp"
+ ],
+ [
+ "Test400/T0271.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0271.bmp"
+ ],
+ [
+ "Test400/T0272.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0272.bmp"
+ ],
+ [
+ "Test400/T0273.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0273.bmp"
+ ],
+ [
+ "Test400/T0274.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0274.bmp"
+ ],
+ [
+ "Test400/T0275.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0275.bmp"
+ ],
+ [
+ "Test400/T0276.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0276.bmp"
+ ],
+ [
+ "Test400/T0277.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0277.bmp"
+ ],
+ [
+ "Test400/T0278.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0278.bmp"
+ ],
+ [
+ "Test400/T0279.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0279.bmp"
+ ],
+ [
+ "Test400/T0280.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0280.bmp"
+ ],
+ [
+ "Test400/T0281.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0281.bmp"
+ ],
+ [
+ "Test400/T0282.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0282.bmp"
+ ],
+ [
+ "Test400/T0283.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0283.bmp"
+ ],
+ [
+ "Test400/T0284.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0284.bmp"
+ ],
+ [
+ "Test400/T0285.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0285.bmp"
+ ],
+ [
+ "Test400/T0286.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0286.bmp"
+ ],
+ [
+ "Test400/T0287.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0287.bmp"
+ ],
+ [
+ "Test400/T0288.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0288.bmp"
+ ],
+ [
+ "Test400/T0289.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0289.bmp"
+ ],
+ [
+ "Test400/T0290.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0290.bmp"
+ ],
+ [
+ "Test400/T0291.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0291.bmp"
+ ],
+ [
+ "Test400/T0292.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0292.bmp"
+ ],
+ [
+ "Test400/T0293.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0293.bmp"
+ ],
+ [
+ "Test400/T0294.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0294.bmp"
+ ],
+ [
+ "Test400/T0295.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0295.bmp"
+ ],
+ [
+ "Test400/T0296.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0296.bmp"
+ ],
+ [
+ "Test400/T0297.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0297.bmp"
+ ],
+ [
+ "Test400/T0298.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0298.bmp"
+ ],
+ [
+ "Test400/T0299.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0299.bmp"
+ ],
+ [
+ "Test400/T0300.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0300.bmp"
+ ],
+ [
+ "Test400/T0301.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0301.bmp"
+ ],
+ [
+ "Test400/T0302.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0302.bmp"
+ ],
+ [
+ "Test400/T0303.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0303.bmp"
+ ],
+ [
+ "Test400/T0304.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0304.bmp"
+ ],
+ [
+ "Test400/T0305.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0305.bmp"
+ ],
+ [
+ "Test400/T0306.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0306.bmp"
+ ],
+ [
+ "Test400/T0307.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0307.bmp"
+ ],
+ [
+ "Test400/T0308.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0308.bmp"
+ ],
+ [
+ "Test400/T0309.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0309.bmp"
+ ],
+ [
+ "Test400/T0310.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0310.bmp"
+ ],
+ [
+ "Test400/T0311.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0311.bmp"
+ ],
+ [
+ "Test400/T0312.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0312.bmp"
+ ],
+ [
+ "Test400/T0313.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0313.bmp"
+ ],
+ [
+ "Test400/T0314.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0314.bmp"
+ ],
+ [
+ "Test400/T0315.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0315.bmp"
+ ],
+ [
+ "Test400/T0316.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0316.bmp"
+ ],
+ [
+ "Test400/T0317.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0317.bmp"
+ ],
+ [
+ "Test400/T0318.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0318.bmp"
+ ],
+ [
+ "Test400/T0319.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0319.bmp"
+ ],
+ [
+ "Test400/T0320.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0320.bmp"
+ ],
+ [
+ "Test400/T0321.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0321.bmp"
+ ],
+ [
+ "Test400/T0322.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0322.bmp"
+ ],
+ [
+ "Test400/T0323.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0323.bmp"
+ ],
+ [
+ "Test400/T0324.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0324.bmp"
+ ],
+ [
+ "Test400/T0325.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0325.bmp"
+ ],
+ [
+ "Test400/T0326.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0326.bmp"
+ ],
+ [
+ "Test400/T0327.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0327.bmp"
+ ],
+ [
+ "Test400/T0328.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0328.bmp"
+ ],
+ [
+ "Test400/T0329.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0329.bmp"
+ ],
+ [
+ "Test400/T0330.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0330.bmp"
+ ],
+ [
+ "Test400/T0331.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0331.bmp"
+ ],
+ [
+ "Test400/T0332.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0332.bmp"
+ ],
+ [
+ "Test400/T0333.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0333.bmp"
+ ],
+ [
+ "Test400/T0334.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0334.bmp"
+ ],
+ [
+ "Test400/T0335.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0335.bmp"
+ ],
+ [
+ "Test400/T0336.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0336.bmp"
+ ],
+ [
+ "Test400/T0337.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0337.bmp"
+ ],
+ [
+ "Test400/T0338.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0338.bmp"
+ ],
+ [
+ "Test400/T0339.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0339.bmp"
+ ],
+ [
+ "Test400/T0340.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0340.bmp"
+ ],
+ [
+ "Test400/T0341.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0341.bmp"
+ ],
+ [
+ "Test400/T0342.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0342.bmp"
+ ],
+ [
+ "Test400/T0343.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0343.bmp"
+ ],
+ [
+ "Test400/T0344.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0344.bmp"
+ ],
+ [
+ "Test400/T0345.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0345.bmp"
+ ],
+ [
+ "Test400/T0346.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0346.bmp"
+ ],
+ [
+ "Test400/T0347.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0347.bmp"
+ ],
+ [
+ "Test400/T0348.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0348.bmp"
+ ],
+ [
+ "Test400/T0349.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0349.bmp"
+ ],
+ [
+ "Test400/T0350.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0350.bmp"
+ ],
+ [
+ "Test400/T0351.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0351.bmp"
+ ],
+ [
+ "Test400/T0352.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0352.bmp"
+ ],
+ [
+ "Test400/T0353.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0353.bmp"
+ ],
+ [
+ "Test400/T0354.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0354.bmp"
+ ],
+ [
+ "Test400/T0355.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0355.bmp"
+ ],
+ [
+ "Test400/T0356.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0356.bmp"
+ ],
+ [
+ "Test400/T0357.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0357.bmp"
+ ],
+ [
+ "Test400/T0358.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0358.bmp"
+ ],
+ [
+ "Test400/T0359.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0359.bmp"
+ ],
+ [
+ "Test400/T0360.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0360.bmp"
+ ],
+ [
+ "Test400/T0361.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0361.bmp"
+ ],
+ [
+ "Test400/T0362.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0362.bmp"
+ ],
+ [
+ "Test400/T0363.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0363.bmp"
+ ],
+ [
+ "Test400/T0364.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0364.bmp"
+ ],
+ [
+ "Test400/T0365.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0365.bmp"
+ ],
+ [
+ "Test400/T0366.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0366.bmp"
+ ],
+ [
+ "Test400/T0367.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0367.bmp"
+ ],
+ [
+ "Test400/T0368.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0368.bmp"
+ ],
+ [
+ "Test400/T0369.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0369.bmp"
+ ],
+ [
+ "Test400/T0370.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0370.bmp"
+ ],
+ [
+ "Test400/T0371.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0371.bmp"
+ ],
+ [
+ "Test400/T0372.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0372.bmp"
+ ],
+ [
+ "Test400/T0373.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0373.bmp"
+ ],
+ [
+ "Test400/T0374.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0374.bmp"
+ ],
+ [
+ "Test400/T0375.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0375.bmp"
+ ],
+ [
+ "Test400/T0376.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0376.bmp"
+ ],
+ [
+ "Test400/T0377.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0377.bmp"
+ ],
+ [
+ "Test400/T0378.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0378.bmp"
+ ],
+ [
+ "Test400/T0379.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0379.bmp"
+ ],
+ [
+ "Test400/T0380.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0380.bmp"
+ ],
+ [
+ "Test400/T0381.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0381.bmp"
+ ],
+ [
+ "Test400/T0382.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0382.bmp"
+ ],
+ [
+ "Test400/T0383.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0383.bmp"
+ ],
+ [
+ "Test400/T0384.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0384.bmp"
+ ],
+ [
+ "Test400/T0385.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0385.bmp"
+ ],
+ [
+ "Test400/T0386.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0386.bmp"
+ ],
+ [
+ "Test400/T0387.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0387.bmp"
+ ],
+ [
+ "Test400/T0388.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0388.bmp"
+ ],
+ [
+ "Test400/T0389.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0389.bmp"
+ ],
+ [
+ "Test400/T0390.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0390.bmp"
+ ],
+ [
+ "Test400/T0391.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0391.bmp"
+ ],
+ [
+ "Test400/T0392.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0392.bmp"
+ ],
+ [
+ "Test400/T0393.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0393.bmp"
+ ],
+ [
+ "Test400/T0394.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0394.bmp"
+ ],
+ [
+ "Test400/T0395.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0395.bmp"
+ ],
+ [
+ "Test400/T0396.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0396.bmp"
+ ],
+ [
+ "Test400/T0397.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/G/T0397.bmp"
+ ],
+ [
+ "Test400/T0398.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0398.bmp"
+ ],
+ [
+ "Test400/T0399.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0399.bmp"
+ ],
+ [
+ "Test400/T0400.jpg",
+ "REFUGE-Test-GT/Disc_Cup_Masks/N/T0400.bmp"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/rimoner3/__init__.py b/bob/ip/binseg/data/rimoner3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e3eca89794709c4cce482b3c2d16565014b8dc1
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/__init__.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""RIM-ONE r3 (training set) for Cup Segmentation
+
+The dataset contains 159 stereo eye fundus images with a resolution of 2144 x
+1424. The right part of the stereo image is disregarded. Two sets of
+ground-truths for optic disc and optic cup are available. The first set is
+commonly used for training and testing. The second set acts as a “humanâ€
+baseline. A third set, composed of annotation averages may also be used for
+training and evaluation purposes.
+
+* Reference: [RIMONER3-2015]_
+* Original resolution (height x width): 1424 x 1072
+* Split reference: [MANINIS-2016]_
+* Protocols ``optic-disc-exp1``, ``optic-cup-exp1``, ``optic-disc-exp2``,
+ ``optic-cup-exp2``, ``optic-disc-avg`` and ``optic-cup-avg``:
+
+ * Training: 99
+ * Test: 60
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "optic-disc-exp1.json"),
+ pkg_resources.resource_filename(__name__, "optic-cup-exp1.json"),
+ pkg_resources.resource_filename(__name__, "optic-disc-exp2.json"),
+ pkg_resources.resource_filename(__name__, "optic-cup-exp2.json"),
+ pkg_resources.resource_filename(__name__, "optic-disc-avg.json"),
+ pkg_resources.resource_filename(__name__, "optic-cup-avg.json"),
+]
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.rimoner3.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _raw_data_loader(sample):
+ # RIM-ONE r3 provides stereo images - we clip them here to get only the
+ # left part of the image, which is also annotated
+ return dict(
+ data=load_pil_rgb(os.path.join(_root_path, sample["data"])).crop(
+ (0, 0, 1072, 1424)
+ ),
+ label=load_pil_1(os.path.join(_root_path, sample["label"])).crop(
+ (0, 0, 1072, 1424)
+ ),
+ )
+
+
+def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # we returned delayed samples to avoid loading all images at once
+ return make_delayed(sample, _raw_data_loader)
+
+
+dataset = JSONDataset(
+ protocols=_protocols,
+ fieldnames=("data", "label"),
+ loader=_loader,
+)
+"""RIM-ONE r3 dataset object"""
diff --git a/bob/ip/binseg/data/rimoner3/optic-cup-avg.json b/bob/ip/binseg/data/rimoner3/optic-cup-avg.json
new file mode 100644
index 0000000000000000000000000000000000000000..d464222e32e8f1da24898a0c8a7308564c9d0a7f
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-cup-avg.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Average_masks/N-10-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Average_masks/N-11-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Average_masks/N-12-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Average_masks/N-13-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Average_masks/N-14-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Average_masks/N-16-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Average_masks/N-24-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Average_masks/N-25-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Average_masks/N-26-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Average_masks/N-27-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Average_masks/N-52-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Average_masks/N-53-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Average_masks/N-54-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Average_masks/N-55-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Average_masks/N-56-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Average_masks/N-58-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Average_masks/N-59-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Average_masks/N-5-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Average_masks/N-61-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Average_masks/N-62-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Average_masks/N-63-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Average_masks/N-64-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Average_masks/N-65-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Average_masks/N-66-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Average_masks/N-67-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Average_masks/N-68-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Average_masks/N-69-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Average_masks/N-6-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Average_masks/N-70-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Average_masks/N-71-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Average_masks/N-72-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Average_masks/N-73-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Average_masks/N-74-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Average_masks/N-75-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Average_masks/N-76-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Average_masks/N-79-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Average_masks/N-7-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Average_masks/N-80-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Average_masks/N-81-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Average_masks/N-82-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Average_masks/N-83-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Average_masks/N-84-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Average_masks/N-85-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Average_masks/N-86-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Average_masks/N-87-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Average_masks/N-88-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Average_masks/N-8-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Average_masks/N-90-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Average_masks/N-91-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Average_masks/N-92-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Average_masks/N-9-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-10-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-11-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-12-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-13-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-14-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-15-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-16-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-17-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-18-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-19-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-1-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-20-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-21-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-32-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-33-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-34-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-35-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-36-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-37-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-38-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-39-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-3-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-4-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-5-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-6-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-7-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-8-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-9-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-10-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-11-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-12-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-13-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-14-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-15-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-16-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-17-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-18-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-19-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-1-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-20-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-21-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-22-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-23-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-24-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-25-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-26-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-27-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-28-R-Cup-Avg.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-22-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-23-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-24-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-25-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-26-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-27-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-28-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-29-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-2-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-30-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-31-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Average_masks/N-17-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Average_masks/N-18-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Average_masks/N-1-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Average_masks/N-20-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Average_masks/N-21-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Average_masks/N-22-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Average_masks/N-23-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Average_masks/N-28-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Average_masks/N-29-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Average_masks/N-2-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Average_masks/N-30-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Average_masks/N-31-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Average_masks/N-32-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Average_masks/N-33-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Average_masks/N-34-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Average_masks/N-35-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Average_masks/N-36-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Average_masks/N-37-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Average_masks/N-38-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Average_masks/N-39-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Average_masks/N-3-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Average_masks/N-40-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Average_masks/N-41-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Average_masks/N-42-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Average_masks/N-43-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Average_masks/N-44-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Average_masks/N-46-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Average_masks/N-47-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Average_masks/N-48-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Average_masks/N-49-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Average_masks/N-4-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Average_masks/N-50-R-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Average_masks/N-51-L-Cup-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Average_masks/N-78-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-29-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-2-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-30-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-31-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-32-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-33-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-34-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-35-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-3-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-4-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-5-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-6-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-7-L-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-8-R-Cup-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-9-L-Cup-Avg.png"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/rimoner3/optic-cup-exp1.json b/bob/ip/binseg/data/rimoner3/optic-cup-exp1.json
new file mode 100644
index 0000000000000000000000000000000000000000..39b14f46763852e652ab5345969a45919fd06f84
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-cup-exp1.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Expert1_masks/N-10-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Expert1_masks/N-11-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Expert1_masks/N-12-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Expert1_masks/N-13-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Expert1_masks/N-14-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Expert1_masks/N-16-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Expert1_masks/N-24-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Expert1_masks/N-25-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Expert1_masks/N-26-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Expert1_masks/N-27-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Expert1_masks/N-52-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Expert1_masks/N-53-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Expert1_masks/N-54-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Expert1_masks/N-55-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Expert1_masks/N-56-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Expert1_masks/N-58-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Expert1_masks/N-59-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Expert1_masks/N-5-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Expert1_masks/N-61-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Expert1_masks/N-62-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Expert1_masks/N-63-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Expert1_masks/N-64-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Expert1_masks/N-65-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Expert1_masks/N-66-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Expert1_masks/N-67-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Expert1_masks/N-68-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Expert1_masks/N-69-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Expert1_masks/N-6-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Expert1_masks/N-70-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Expert1_masks/N-71-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Expert1_masks/N-72-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Expert1_masks/N-73-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Expert1_masks/N-74-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Expert1_masks/N-75-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Expert1_masks/N-76-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Expert1_masks/N-79-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Expert1_masks/N-7-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Expert1_masks/N-80-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Expert1_masks/N-81-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Expert1_masks/N-82-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Expert1_masks/N-83-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Expert1_masks/N-84-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Expert1_masks/N-85-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Expert1_masks/N-86-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Expert1_masks/N-87-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Expert1_masks/N-88-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Expert1_masks/N-8-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Expert1_masks/N-90-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Expert1_masks/N-91-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Expert1_masks/N-92-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Expert1_masks/N-9-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-10-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-11-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-12-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-13-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-14-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-15-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-16-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-17-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-18-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-19-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-1-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-20-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-21-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-32-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-33-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-34-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-35-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-36-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-37-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-38-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-39-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-3-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-4-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-5-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-6-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-7-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-8-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-9-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-10-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-11-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-12-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-13-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-14-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-15-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-16-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-17-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-18-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-19-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-1-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-20-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-21-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-22-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-23-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-24-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-25-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-26-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-27-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-28-R-1-Cup-exp1.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-22-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-23-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-24-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-25-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-26-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-27-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-28-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-29-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-2-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-30-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-31-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Expert1_masks/N-17-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Expert1_masks/N-18-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Expert1_masks/N-1-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Expert1_masks/N-20-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Expert1_masks/N-21-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Expert1_masks/N-22-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Expert1_masks/N-23-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Expert1_masks/N-28-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Expert1_masks/N-29-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Expert1_masks/N-2-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Expert1_masks/N-30-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Expert1_masks/N-31-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Expert1_masks/N-32-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Expert1_masks/N-33-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Expert1_masks/N-34-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Expert1_masks/N-35-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Expert1_masks/N-36-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Expert1_masks/N-37-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Expert1_masks/N-38-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Expert1_masks/N-39-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Expert1_masks/N-3-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Expert1_masks/N-40-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Expert1_masks/N-41-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Expert1_masks/N-42-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Expert1_masks/N-43-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Expert1_masks/N-44-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Expert1_masks/N-46-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Expert1_masks/N-47-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Expert1_masks/N-48-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Expert1_masks/N-49-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Expert1_masks/N-4-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Expert1_masks/N-50-R-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Expert1_masks/N-51-L-1-Cup-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Expert1_masks/N-78-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-29-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-2-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-30-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-31-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-32-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-33-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-34-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-35-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-3-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-4-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-5-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-6-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-7-L-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-8-R-1-Cup-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-9-L-1-Cup-exp1.png"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/rimoner3/optic-cup-exp2.json b/bob/ip/binseg/data/rimoner3/optic-cup-exp2.json
new file mode 100644
index 0000000000000000000000000000000000000000..32b9850155d9a3d895d2f41214dff0402b8d542c
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-cup-exp2.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Expert2_masks/N-10-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Expert2_masks/N-11-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Expert2_masks/N-12-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Expert2_masks/N-13-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Expert2_masks/N-14-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Expert2_masks/N-16-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Expert2_masks/N-24-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Expert2_masks/N-25-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Expert2_masks/N-26-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Expert2_masks/N-27-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Expert2_masks/N-52-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Expert2_masks/N-53-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Expert2_masks/N-54-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Expert2_masks/N-55-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Expert2_masks/N-56-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Expert2_masks/N-58-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Expert2_masks/N-59-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Expert2_masks/N-5-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Expert2_masks/N-61-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Expert2_masks/N-62-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Expert2_masks/N-63-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Expert2_masks/N-64-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Expert2_masks/N-65-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Expert2_masks/N-66-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Expert2_masks/N-67-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Expert2_masks/N-68-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Expert2_masks/N-69-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Expert2_masks/N-6-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Expert2_masks/N-70-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Expert2_masks/N-71-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Expert2_masks/N-72-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Expert2_masks/N-73-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Expert2_masks/N-74-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Expert2_masks/N-75-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Expert2_masks/N-76-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Expert2_masks/N-79-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Expert2_masks/N-7-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Expert2_masks/N-80-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Expert2_masks/N-81-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Expert2_masks/N-82-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Expert2_masks/N-83-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Expert2_masks/N-84-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Expert2_masks/N-85-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Expert2_masks/N-86-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Expert2_masks/N-87-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Expert2_masks/N-88-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Expert2_masks/N-8-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Expert2_masks/N-90-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Expert2_masks/N-91-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Expert2_masks/N-92-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Expert2_masks/N-9-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-10-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-11-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-12-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-13-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-14-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-15-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-16-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-17-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-18-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-19-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-1-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-20-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-21-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-32-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-33-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-34-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-35-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-36-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-37-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-38-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-39-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-3-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-4-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-5-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-6-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-7-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-8-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-9-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-10-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-11-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-12-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-13-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-14-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-15-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-16-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-17-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-18-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-19-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-1-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-20-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-21-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-22-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-23-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-24-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-25-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-26-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-27-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-28-R-1-Cup-exp2.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-22-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-23-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-24-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-25-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-26-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-27-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-28-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-29-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-2-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-30-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-31-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Expert2_masks/N-17-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Expert2_masks/N-18-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Expert2_masks/N-1-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Expert2_masks/N-20-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Expert2_masks/N-21-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Expert2_masks/N-22-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Expert2_masks/N-23-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Expert2_masks/N-28-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Expert2_masks/N-29-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Expert2_masks/N-2-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Expert2_masks/N-30-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Expert2_masks/N-31-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Expert2_masks/N-32-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Expert2_masks/N-33-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Expert2_masks/N-34-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Expert2_masks/N-35-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Expert2_masks/N-36-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Expert2_masks/N-37-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Expert2_masks/N-38-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Expert2_masks/N-39-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Expert2_masks/N-3-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Expert2_masks/N-40-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Expert2_masks/N-41-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Expert2_masks/N-42-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Expert2_masks/N-43-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Expert2_masks/N-44-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Expert2_masks/N-46-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Expert2_masks/N-47-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Expert2_masks/N-48-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Expert2_masks/N-49-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Expert2_masks/N-4-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Expert2_masks/N-50-R-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Expert2_masks/N-51-L-1-Cup-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Expert2_masks/N-78-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-29-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-2-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-30-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-31-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-32-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-33-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-34-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-35-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-3-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-4-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-5-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-6-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-7-L-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-8-R-1-Cup-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-9-L-1-Cup-exp2.png"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/rimoner3/optic-disc-avg.json b/bob/ip/binseg/data/rimoner3/optic-disc-avg.json
new file mode 100644
index 0000000000000000000000000000000000000000..987d06c04a3b47148734af7003fdfa86101a34ad
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-disc-avg.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Average_masks/N-10-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Average_masks/N-11-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Average_masks/N-12-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Average_masks/N-13-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Average_masks/N-14-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Average_masks/N-16-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Average_masks/N-24-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Average_masks/N-25-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Average_masks/N-26-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Average_masks/N-27-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Average_masks/N-52-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Average_masks/N-53-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Average_masks/N-54-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Average_masks/N-55-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Average_masks/N-56-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Average_masks/N-58-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Average_masks/N-59-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Average_masks/N-5-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Average_masks/N-61-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Average_masks/N-62-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Average_masks/N-63-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Average_masks/N-64-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Average_masks/N-65-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Average_masks/N-66-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Average_masks/N-67-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Average_masks/N-68-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Average_masks/N-69-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Average_masks/N-6-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Average_masks/N-70-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Average_masks/N-71-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Average_masks/N-72-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Average_masks/N-73-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Average_masks/N-74-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Average_masks/N-75-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Average_masks/N-76-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Average_masks/N-79-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Average_masks/N-7-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Average_masks/N-80-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Average_masks/N-81-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Average_masks/N-82-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Average_masks/N-83-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Average_masks/N-84-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Average_masks/N-85-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Average_masks/N-86-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Average_masks/N-87-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Average_masks/N-88-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Average_masks/N-8-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Average_masks/N-90-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Average_masks/N-91-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Average_masks/N-92-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Average_masks/N-9-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-10-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-11-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-12-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-13-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-14-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-15-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-16-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-17-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-18-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-19-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-1-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-20-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-21-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-32-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-33-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-34-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-35-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-36-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-37-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-38-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-39-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-3-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-4-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-5-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-6-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-7-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-8-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-9-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-10-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-11-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-12-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-13-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-14-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-15-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-16-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-17-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-18-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-19-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-1-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-20-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-21-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-22-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-23-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-24-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-25-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-26-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-27-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-28-R-Disc-Avg.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-22-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-23-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-24-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-25-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-26-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-27-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-28-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-29-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-2-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Average_masks/G-30-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Average_masks/G-31-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Average_masks/N-17-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Average_masks/N-18-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Average_masks/N-1-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Average_masks/N-20-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Average_masks/N-21-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Average_masks/N-22-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Average_masks/N-23-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Average_masks/N-28-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Average_masks/N-29-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Average_masks/N-2-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Average_masks/N-30-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Average_masks/N-31-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Average_masks/N-32-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Average_masks/N-33-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Average_masks/N-34-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Average_masks/N-35-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Average_masks/N-36-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Average_masks/N-37-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Average_masks/N-38-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Average_masks/N-39-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Average_masks/N-3-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Average_masks/N-40-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Average_masks/N-41-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Average_masks/N-42-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Average_masks/N-43-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Average_masks/N-44-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Average_masks/N-46-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Average_masks/N-47-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Average_masks/N-48-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Average_masks/N-49-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Average_masks/N-4-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Average_masks/N-50-R-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Average_masks/N-51-L-Disc-Avg.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Average_masks/N-78-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-29-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-2-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-30-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-31-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-32-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-33-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-34-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-35-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-3-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-4-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-5-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-6-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-7-L-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Average_masks/S-8-R-Disc-Avg.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Average_masks/S-9-L-Disc-Avg.png"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/rimoner3/optic-disc-exp1.json b/bob/ip/binseg/data/rimoner3/optic-disc-exp1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c5a8654ca0f28ae34fe615e98925641c868afe1c
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-disc-exp1.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Expert1_masks/N-10-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Expert1_masks/N-11-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Expert1_masks/N-12-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Expert1_masks/N-13-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Expert1_masks/N-14-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Expert1_masks/N-16-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Expert1_masks/N-24-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Expert1_masks/N-25-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Expert1_masks/N-26-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Expert1_masks/N-27-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Expert1_masks/N-52-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Expert1_masks/N-53-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Expert1_masks/N-54-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Expert1_masks/N-55-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Expert1_masks/N-56-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Expert1_masks/N-58-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Expert1_masks/N-59-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Expert1_masks/N-5-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Expert1_masks/N-61-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Expert1_masks/N-62-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Expert1_masks/N-63-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Expert1_masks/N-64-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Expert1_masks/N-65-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Expert1_masks/N-66-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Expert1_masks/N-67-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Expert1_masks/N-68-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Expert1_masks/N-69-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Expert1_masks/N-6-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Expert1_masks/N-70-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Expert1_masks/N-71-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Expert1_masks/N-72-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Expert1_masks/N-73-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Expert1_masks/N-74-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Expert1_masks/N-75-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Expert1_masks/N-76-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Expert1_masks/N-79-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Expert1_masks/N-7-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Expert1_masks/N-80-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Expert1_masks/N-81-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Expert1_masks/N-82-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Expert1_masks/N-83-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Expert1_masks/N-84-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Expert1_masks/N-85-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Expert1_masks/N-86-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Expert1_masks/N-87-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Expert1_masks/N-88-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Expert1_masks/N-8-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Expert1_masks/N-90-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Expert1_masks/N-91-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Expert1_masks/N-92-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Expert1_masks/N-9-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-10-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-11-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-12-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-13-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-14-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-15-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-16-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-17-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-18-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-19-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-1-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-20-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-21-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-32-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-33-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-34-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-35-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-36-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-37-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-38-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-39-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-3-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-4-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-5-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-6-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-7-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-8-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-9-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-10-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-11-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-12-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-13-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-14-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-15-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-16-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-17-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-18-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-19-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-1-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-20-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-21-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-22-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-23-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-24-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-25-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-26-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-27-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-28-R-1-Disc-exp1.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-22-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-23-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-24-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-25-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-26-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-27-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-28-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-29-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-2-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-30-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/G-31-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Expert1_masks/N-17-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Expert1_masks/N-18-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Expert1_masks/N-1-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Expert1_masks/N-20-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Expert1_masks/N-21-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Expert1_masks/N-22-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Expert1_masks/N-23-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Expert1_masks/N-28-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Expert1_masks/N-29-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Expert1_masks/N-2-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Expert1_masks/N-30-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Expert1_masks/N-31-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Expert1_masks/N-32-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Expert1_masks/N-33-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Expert1_masks/N-34-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Expert1_masks/N-35-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Expert1_masks/N-36-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Expert1_masks/N-37-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Expert1_masks/N-38-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Expert1_masks/N-39-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Expert1_masks/N-3-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Expert1_masks/N-40-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Expert1_masks/N-41-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Expert1_masks/N-42-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Expert1_masks/N-43-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Expert1_masks/N-44-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Expert1_masks/N-46-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Expert1_masks/N-47-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Expert1_masks/N-48-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Expert1_masks/N-49-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Expert1_masks/N-4-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Expert1_masks/N-50-R-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Expert1_masks/N-51-L-1-Disc-exp1.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Expert1_masks/N-78-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-29-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-2-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-30-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-31-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-32-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-33-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-34-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-35-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-3-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-4-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-5-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-6-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-7-L-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-8-R-1-Disc-exp1.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Expert1_masks/S-9-L-1-Disc-exp1.png"
+ ]
+ ]
+}
\ No newline at end of file
diff --git a/bob/ip/binseg/data/rimoner3/optic-disc-exp2.json b/bob/ip/binseg/data/rimoner3/optic-disc-exp2.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a94781e740e06cdc47cdd1a0e07070ef9d7eb5b
--- /dev/null
+++ b/bob/ip/binseg/data/rimoner3/optic-disc-exp2.json
@@ -0,0 +1,642 @@
+{
+ "train": [
+ [
+ "Healthy/Stereo Images/N-10-R.jpg",
+ "Healthy/Expert2_masks/N-10-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-11-L.jpg",
+ "Healthy/Expert2_masks/N-11-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-12-R.jpg",
+ "Healthy/Expert2_masks/N-12-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-13-L.jpg",
+ "Healthy/Expert2_masks/N-13-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-14-R.jpg",
+ "Healthy/Expert2_masks/N-14-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-16-R.jpg",
+ "Healthy/Expert2_masks/N-16-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-24-R.jpg",
+ "Healthy/Expert2_masks/N-24-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-25-L.jpg",
+ "Healthy/Expert2_masks/N-25-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-26-R.jpg",
+ "Healthy/Expert2_masks/N-26-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-27-L.jpg",
+ "Healthy/Expert2_masks/N-27-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-52-R.jpg",
+ "Healthy/Expert2_masks/N-52-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-53-L.jpg",
+ "Healthy/Expert2_masks/N-53-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-54-R.jpg",
+ "Healthy/Expert2_masks/N-54-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-55-L.jpg",
+ "Healthy/Expert2_masks/N-55-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-56-R.jpg",
+ "Healthy/Expert2_masks/N-56-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-58-R.jpg",
+ "Healthy/Expert2_masks/N-58-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-59-L.jpg",
+ "Healthy/Expert2_masks/N-59-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-5-L.jpg",
+ "Healthy/Expert2_masks/N-5-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-61-L.jpg",
+ "Healthy/Expert2_masks/N-61-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-62-R.jpg",
+ "Healthy/Expert2_masks/N-62-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-63-L.jpg",
+ "Healthy/Expert2_masks/N-63-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-64-R.jpg",
+ "Healthy/Expert2_masks/N-64-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-65-L.jpg",
+ "Healthy/Expert2_masks/N-65-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-66-R.jpg",
+ "Healthy/Expert2_masks/N-66-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-67-L.jpg",
+ "Healthy/Expert2_masks/N-67-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-68-R.jpg",
+ "Healthy/Expert2_masks/N-68-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-69-L.jpg",
+ "Healthy/Expert2_masks/N-69-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-6-R.jpg",
+ "Healthy/Expert2_masks/N-6-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-70-R.jpg",
+ "Healthy/Expert2_masks/N-70-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-71-L.jpg",
+ "Healthy/Expert2_masks/N-71-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-72-R.jpg",
+ "Healthy/Expert2_masks/N-72-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-73-R.jpg",
+ "Healthy/Expert2_masks/N-73-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-74-L.jpg",
+ "Healthy/Expert2_masks/N-74-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-75-R.jpg",
+ "Healthy/Expert2_masks/N-75-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-76-R.jpg",
+ "Healthy/Expert2_masks/N-76-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-79-L.jpg",
+ "Healthy/Expert2_masks/N-79-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-7-L.jpg",
+ "Healthy/Expert2_masks/N-7-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-80-R.jpg",
+ "Healthy/Expert2_masks/N-80-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-81-L.jpg",
+ "Healthy/Expert2_masks/N-81-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-82-R.jpg",
+ "Healthy/Expert2_masks/N-82-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-83-L.jpg",
+ "Healthy/Expert2_masks/N-83-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-84-R.jpg",
+ "Healthy/Expert2_masks/N-84-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-85-L.jpg",
+ "Healthy/Expert2_masks/N-85-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-86-R.jpg",
+ "Healthy/Expert2_masks/N-86-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-87-L.jpg",
+ "Healthy/Expert2_masks/N-87-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-88-R.jpg",
+ "Healthy/Expert2_masks/N-88-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-8-L.jpg",
+ "Healthy/Expert2_masks/N-8-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-90-R.jpg",
+ "Healthy/Expert2_masks/N-90-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-91-L.jpg",
+ "Healthy/Expert2_masks/N-91-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-92-R.jpg",
+ "Healthy/Expert2_masks/N-92-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-9-L.jpg",
+ "Healthy/Expert2_masks/N-9-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-10-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-10-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-11-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-11-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-12-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-12-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-13-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-13-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-14-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-14-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-15-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-15-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-16-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-16-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-17-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-17-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-18-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-18-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-19-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-19-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-1-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-1-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-20-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-20-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-21-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-21-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-32-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-32-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-33-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-33-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-34-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-34-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-35-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-35-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-36-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-36-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-37-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-37-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-38-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-38-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-39-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-39-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-3-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-3-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-4-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-4-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-5-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-5-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-6-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-6-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-7-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-7-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-8-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-8-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-9-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-9-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-10-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-10-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-11-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-11-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-12-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-12-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-13-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-13-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-14-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-14-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-15-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-15-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-16-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-16-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-17-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-17-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-18-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-18-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-19-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-19-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-1-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-1-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-20-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-20-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-21-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-21-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-22-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-22-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-23-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-23-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-24-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-24-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-25-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-25-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-26-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-26-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-27-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-27-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-28-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-28-R-1-Disc-exp2.png"
+ ]
+ ],
+ "test": [
+ [
+ "Glaucoma and suspects/Stereo Images/G-22-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-22-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-23-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-23-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-24-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-24-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-25-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-25-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-26-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-26-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-27-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-27-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-28-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-28-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-29-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-29-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-2-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-2-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-30-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-30-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/G-31-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/G-31-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-17-L.jpg",
+ "Healthy/Expert2_masks/N-17-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-18-R.jpg",
+ "Healthy/Expert2_masks/N-18-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-1-L.jpg",
+ "Healthy/Expert2_masks/N-1-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-20-R.jpg",
+ "Healthy/Expert2_masks/N-20-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-21-L.jpg",
+ "Healthy/Expert2_masks/N-21-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-22-R.jpg",
+ "Healthy/Expert2_masks/N-22-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-23-L.jpg",
+ "Healthy/Expert2_masks/N-23-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-28-R.jpg",
+ "Healthy/Expert2_masks/N-28-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-29-L.jpg",
+ "Healthy/Expert2_masks/N-29-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-2-R.jpg",
+ "Healthy/Expert2_masks/N-2-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-30-R.jpg",
+ "Healthy/Expert2_masks/N-30-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-31-L.jpg",
+ "Healthy/Expert2_masks/N-31-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-32-R.jpg",
+ "Healthy/Expert2_masks/N-32-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-33-L.jpg",
+ "Healthy/Expert2_masks/N-33-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-34-R.jpg",
+ "Healthy/Expert2_masks/N-34-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-35-L.jpg",
+ "Healthy/Expert2_masks/N-35-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-36-R.jpg",
+ "Healthy/Expert2_masks/N-36-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-37-L.jpg",
+ "Healthy/Expert2_masks/N-37-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-38-R.jpg",
+ "Healthy/Expert2_masks/N-38-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-39-L.jpg",
+ "Healthy/Expert2_masks/N-39-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-3-L.jpg",
+ "Healthy/Expert2_masks/N-3-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-40-R.jpg",
+ "Healthy/Expert2_masks/N-40-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-41-L.jpg",
+ "Healthy/Expert2_masks/N-41-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-42-R.jpg",
+ "Healthy/Expert2_masks/N-42-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-43-L.jpg",
+ "Healthy/Expert2_masks/N-43-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-44-R.jpg",
+ "Healthy/Expert2_masks/N-44-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-46-R.jpg",
+ "Healthy/Expert2_masks/N-46-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-47-L.jpg",
+ "Healthy/Expert2_masks/N-47-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-48-R.jpg",
+ "Healthy/Expert2_masks/N-48-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-49-L.jpg",
+ "Healthy/Expert2_masks/N-49-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-4-R.jpg",
+ "Healthy/Expert2_masks/N-4-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-50-R.jpg",
+ "Healthy/Expert2_masks/N-50-R-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-51-L.jpg",
+ "Healthy/Expert2_masks/N-51-L-1-Disc-exp2.png"
+ ],
+ [
+ "Healthy/Stereo Images/N-78-R.jpg",
+ "Healthy/Expert2_masks/N-78-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-29-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-29-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-2-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-2-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-30-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-30-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-31-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-31-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-32-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-32-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-33-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-33-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-34-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-34-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-35-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-35-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-3-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-3-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-4-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-4-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-5-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-5-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-6-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-6-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-7-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-7-L-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-8-R.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-8-R-1-Disc-exp2.png"
+ ],
+ [
+ "Glaucoma and suspects/Stereo Images/S-9-L.jpg",
+ "Glaucoma and suspects/Expert2_masks/S-9-L-1-Disc-exp2.png"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/sample.py b/bob/ip/binseg/data/sample.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d85bf35b4d548d0ea25aeda194dc1bc33bb8a2f
--- /dev/null
+++ b/bob/ip/binseg/data/sample.py
@@ -0,0 +1,110 @@
+from collections.abc import MutableSequence
+
+"""Base definition of sample
+
+.. todo::
+
+ Copied from bob/bob.pipelines **TEMPORARILY**! Remove this and use the
+ package directly!
+
+"""
+
+
+def _copy_attributes(s, d):
+ """Copies attributes from a dictionary to self
+ """
+ s.__dict__.update(
+ dict([k, v] for k, v in d.items() if k not in ("data", "load", "samples"))
+ )
+
+
+class DelayedSample:
+ """Representation of sample that can be loaded via a callable
+
+ The optional ``**kwargs`` argument allows you to attach more attributes to
+ this sample instance.
+
+
+ Parameters
+ ----------
+
+ load : object
+ A python function that can be called parameterlessly, to load the
+ sample in question from whatever medium
+
+ parent : :py:class:`DelayedSample`, :py:class:`Sample`, None
+ If passed, consider this as a parent of this sample, to copy
+ information
+
+ kwargs : dict
+ Further attributes of this sample, to be stored and eventually
+ transmitted to transformed versions of the sample
+
+ """
+
+ def __init__(self, load, parent=None, **kwargs):
+ self.load = load
+ if parent is not None:
+ _copy_attributes(self, parent.__dict__)
+ _copy_attributes(self, kwargs)
+
+ @property
+ def data(self):
+ """Loads the data from the disk file"""
+ return self.load()
+
+
+class Sample:
+ """Representation of sample that is sufficient for the blocks in this module
+
+ Each sample must have the following attributes:
+
+ * attribute ``data``: Contains the data for this sample
+
+
+ Parameters
+ ----------
+
+ data : object
+ Object representing the data to initialize this sample with.
+
+ parent : object
+ A parent object from which to inherit all other attributes (except
+ ``data``)
+
+ """
+
+ def __init__(self, data, parent=None, **kwargs):
+ self.data = data
+ if parent is not None:
+ _copy_attributes(self, parent.__dict__)
+ _copy_attributes(self, kwargs)
+
+
+
+class SampleSet(MutableSequence):
+ """A set of samples with extra attributes
+ https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes
+ """
+
+ def __init__(self, samples, parent=None, **kwargs):
+ self.samples = samples
+ if parent is not None:
+ _copy_attributes(self, parent.__dict__)
+ _copy_attributes(self, kwargs)
+
+ def __len__(self):
+ return len(self.samples)
+
+ def __getitem__(self, item):
+ return self.samples.__getitem__(item)
+
+ def __setitem__(self, key, item):
+ return self.samples.__setitem__(key, item)
+
+ def __delitem__(self, item):
+ return self.samples.__delitem__(item)
+
+ def insert(self, index, item):
+ # if not item in self.samples:
+ self.samples.insert(index, item)
diff --git a/bob/ip/binseg/data/stare/__init__.py b/bob/ip/binseg/data/stare/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..753c98feb92884f4f3ad11de3cc781a99d1bdb9e
--- /dev/null
+++ b/bob/ip/binseg/data/stare/__init__.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""STARE dataset for Vessel Segmentation
+
+A subset of the original STARE dataset contains 20 annotated eye fundus images
+with a resolution of 700 x 605 (width x height). Two sets of ground-truth
+vessel annotations are available. The first set by Adam Hoover ("ah") is
+commonly used for training and testing. The second set by Valentina Kouznetsova
+("vk") is typically used as a “human†baseline.
+
+* Reference: [STARE-2000]_
+* Original resolution (width x height): 700 x 605
+* Split reference: [MANINIS-2016]_
+* Protocol ``ah`` (default baseline):
+
+ * Training samples: 10 (including labels from annotator "ah")
+ * Test samples: 10 (including labels from annotator "ah")
+
+* Protocol ``vk`` (normally used as human comparison):
+
+ * Training samples: 10 (including labels from annotator "vk")
+ * Test samples: 10 (including labels from annotator "vk")
+
+"""
+
+import os
+import pkg_resources
+
+import bob.extension
+
+from ..dataset import JSONDataset
+from ..loader import load_pil_rgb, load_pil_1, make_delayed
+
+_protocols = [
+ pkg_resources.resource_filename(__name__, "ah.json"),
+ pkg_resources.resource_filename(__name__, "vk.json"),
+]
+
+_fieldnames = ("data", "label")
+
+_root_path = bob.extension.rc.get(
+ "bob.ip.binseg.stare.datadir", os.path.realpath(os.curdir)
+)
+
+
+def _make_loader(root_path):
+ #hack to get testing on the CI working fine for this dataset
+
+ def _raw_data_loader(sample):
+ return dict(
+ data=load_pil_rgb(os.path.join(root_path, sample["data"])),
+ label=load_pil_1(os.path.join(root_path, sample["label"])),
+ )
+
+ def _loader(context, sample):
+ # "context" is ignored in this case - database is homogeneous
+ # we returned delayed samples to avoid loading all images at once
+ return make_delayed(sample, _raw_data_loader)
+
+ return _loader
+
+
+def _make_dataset(root_path):
+
+ return JSONDataset(
+ protocols=_protocols,
+ fieldnames=_fieldnames,
+ loader=_make_loader(root_path),
+ )
+
+dataset = _make_dataset(_root_path)
+"""STARE dataset object"""
diff --git a/bob/ip/binseg/data/stare/ah.json b/bob/ip/binseg/data/stare/ah.json
new file mode 100644
index 0000000000000000000000000000000000000000..38a965944c32d96bfe7e54bf7ca94eaf0a19dcc8
--- /dev/null
+++ b/bob/ip/binseg/data/stare/ah.json
@@ -0,0 +1,86 @@
+{
+ "train": [
+ [
+ "stare-images/im0001.ppm",
+ "labels-ah/im0001.ah.ppm"
+ ],
+ [
+ "stare-images/im0002.ppm",
+ "labels-ah/im0002.ah.ppm"
+ ],
+ [
+ "stare-images/im0003.ppm",
+ "labels-ah/im0003.ah.ppm"
+ ],
+ [
+ "stare-images/im0004.ppm",
+ "labels-ah/im0004.ah.ppm"
+ ],
+ [
+ "stare-images/im0005.ppm",
+ "labels-ah/im0005.ah.ppm"
+ ],
+ [
+ "stare-images/im0044.ppm",
+ "labels-ah/im0044.ah.ppm"
+ ],
+ [
+ "stare-images/im0077.ppm",
+ "labels-ah/im0077.ah.ppm"
+ ],
+ [
+ "stare-images/im0081.ppm",
+ "labels-ah/im0081.ah.ppm"
+ ],
+ [
+ "stare-images/im0082.ppm",
+ "labels-ah/im0082.ah.ppm"
+ ],
+ [
+ "stare-images/im0139.ppm",
+ "labels-ah/im0139.ah.ppm"
+ ]
+ ],
+ "test": [
+ [
+ "stare-images/im0162.ppm",
+ "labels-ah/im0162.ah.ppm"
+ ],
+ [
+ "stare-images/im0163.ppm",
+ "labels-ah/im0163.ah.ppm"
+ ],
+ [
+ "stare-images/im0235.ppm",
+ "labels-ah/im0235.ah.ppm"
+ ],
+ [
+ "stare-images/im0236.ppm",
+ "labels-ah/im0236.ah.ppm"
+ ],
+ [
+ "stare-images/im0239.ppm",
+ "labels-ah/im0239.ah.ppm"
+ ],
+ [
+ "stare-images/im0240.ppm",
+ "labels-ah/im0240.ah.ppm"
+ ],
+ [
+ "stare-images/im0255.ppm",
+ "labels-ah/im0255.ah.ppm"
+ ],
+ [
+ "stare-images/im0291.ppm",
+ "labels-ah/im0291.ah.ppm"
+ ],
+ [
+ "stare-images/im0319.ppm",
+ "labels-ah/im0319.ah.ppm"
+ ],
+ [
+ "stare-images/im0324.ppm",
+ "labels-ah/im0324.ah.ppm"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/stare/vk.json b/bob/ip/binseg/data/stare/vk.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d9d4e464a13de83ca164b4d02e99e7704e0857b
--- /dev/null
+++ b/bob/ip/binseg/data/stare/vk.json
@@ -0,0 +1,86 @@
+{
+ "train": [
+ [
+ "stare-images/im0001.ppm",
+ "labels-vk/im0001.vk.ppm"
+ ],
+ [
+ "stare-images/im0002.ppm",
+ "labels-vk/im0002.vk.ppm"
+ ],
+ [
+ "stare-images/im0003.ppm",
+ "labels-vk/im0003.vk.ppm"
+ ],
+ [
+ "stare-images/im0004.ppm",
+ "labels-vk/im0004.vk.ppm"
+ ],
+ [
+ "stare-images/im0005.ppm",
+ "labels-vk/im0005.vk.ppm"
+ ],
+ [
+ "stare-images/im0044.ppm",
+ "labels-vk/im0044.vk.ppm"
+ ],
+ [
+ "stare-images/im0077.ppm",
+ "labels-vk/im0077.vk.ppm"
+ ],
+ [
+ "stare-images/im0081.ppm",
+ "labels-vk/im0081.vk.ppm"
+ ],
+ [
+ "stare-images/im0082.ppm",
+ "labels-vk/im0082.vk.ppm"
+ ],
+ [
+ "stare-images/im0139.ppm",
+ "labels-vk/im0139.vk.ppm"
+ ]
+ ],
+ "test": [
+ [
+ "stare-images/im0162.ppm",
+ "labels-vk/im0162.vk.ppm"
+ ],
+ [
+ "stare-images/im0163.ppm",
+ "labels-vk/im0163.vk.ppm"
+ ],
+ [
+ "stare-images/im0235.ppm",
+ "labels-vk/im0235.vk.ppm"
+ ],
+ [
+ "stare-images/im0236.ppm",
+ "labels-vk/im0236.vk.ppm"
+ ],
+ [
+ "stare-images/im0239.ppm",
+ "labels-vk/im0239.vk.ppm"
+ ],
+ [
+ "stare-images/im0240.ppm",
+ "labels-vk/im0240.vk.ppm"
+ ],
+ [
+ "stare-images/im0255.ppm",
+ "labels-vk/im0255.vk.ppm"
+ ],
+ [
+ "stare-images/im0291.ppm",
+ "labels-vk/im0291.vk.ppm"
+ ],
+ [
+ "stare-images/im0319.ppm",
+ "labels-vk/im0319.vk.ppm"
+ ],
+ [
+ "stare-images/im0324.ppm",
+ "labels-vk/im0324.vk.ppm"
+ ]
+ ]
+}
diff --git a/bob/ip/binseg/data/transforms.py b/bob/ip/binseg/data/transforms.py
index b97fe7959ba95e571462d6f6d9d2ddade9fad226..bea81bcd9dbf7ba998795f96c35ccae3b7c1b417 100644
--- a/bob/ip/binseg/data/transforms.py
+++ b/bob/ip/binseg/data/transforms.py
@@ -1,76 +1,57 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import torchvision.transforms.functional as VF
+"""Image transformations for our pipelines
+
+Differences between methods here and those from
+:py:mod:`torchvision.transforms` is that these support multiple simultaneous
+image inputs, which are required to feed segmentation networks (e.g. image and
+labels or masks). We also take care of data augmentations, in which random
+flipping and rotation needs to be applied across all input images, but color
+jittering, for example, only on the input image.
+"""
+
import random
-import PIL
-from PIL import Image
-from torchvision.transforms.transforms import Lambda
-from torchvision.transforms.transforms import Compose as TorchVisionCompose
-import math
-from math import floor
-import warnings
-import collections
-import bob.core
-
-_pil_interpolation_to_str = {
- Image.NEAREST: 'PIL.Image.NEAREST',
- Image.BILINEAR: 'PIL.Image.BILINEAR',
- Image.BICUBIC: 'PIL.Image.BICUBIC',
- Image.LANCZOS: 'PIL.Image.LANCZOS',
- Image.HAMMING: 'PIL.Image.HAMMING',
- Image.BOX: 'PIL.Image.BOX',
-}
-Iterable = collections.abc.Iterable
-
-# Compose
-
-class Compose:
- """Composes several transforms.
- Attributes
- ----------
- transforms : list
- list of transforms to compose.
- """
+import numpy
+import PIL.Image
+import torchvision.transforms
+import torchvision.transforms.functional
+
- def __init__(self, transforms):
- self.transforms = transforms
+class TupleMixin:
+ """Adds support to work with tuples of objects to torchvision transforms"""
def __call__(self, *args):
- for t in self.transforms:
- args = t(*args)
- return args
+ return [super(TupleMixin, self).__call__(k) for k in args]
- def __repr__(self):
- format_string = self.__class__.__name__ + '('
- for t in self.transforms:
- format_string += '\n'
- format_string += ' {0}'.format(t)
- format_string += '\n)'
- return format_string
-# Preprocessing
+class CenterCrop(TupleMixin, torchvision.transforms.CenterCrop):
+ pass
-class CenterCrop:
- """
- Crop at the center.
- Attributes
- ----------
- size : int
- target size
- """
- def __init__(self, size):
- self.size = size
+class Pad(TupleMixin, torchvision.transforms.Pad):
+ pass
+
+
+class Resize(TupleMixin, torchvision.transforms.Resize):
+ pass
+
+
+class ToTensor(TupleMixin, torchvision.transforms.ToTensor):
+ pass
+
+class Compose(torchvision.transforms.Compose):
def __call__(self, *args):
- return [VF.center_crop(img, self.size) for img in args]
+ for t in self.transforms:
+ args = t(*args)
+ return args
-class Crop:
+class SingleCrop:
"""
- Crop at the given coordinates.
+ Crops one image at the given coordinates.
Attributes
----------
@@ -83,304 +64,205 @@ class Crop:
w : int
width of the cropped image.
"""
+
def __init__(self, i, j, h, w):
self.i = i
self.j = j
self.h = h
self.w = w
- def __call__(self, *args):
- return [img.crop((self.j, self.i, self.j + self.w, self.i + self.h)) for img in args]
+ def __call__(self, img):
+ return img.crop((self.j, self.i, self.j + self.w, self.i + self.h))
-class Pad:
+
+class Crop(TupleMixin, SingleCrop):
"""
- Constant padding
+ Crops multiple images at the given coordinates.
Attributes
----------
- padding : int or tuple
- padding on each border. If a single int is provided this is used to pad all borders.
- If tuple of length 2 is provided this is the padding on left/right and top/bottom respectively.
- If a tuple of length 4 is provided this is the padding for the left, top, right and bottom borders respectively.
-
- fill : int
- pixel fill value for constant fill. Default is 0. If a tuple of length 3, it is used to fill R, G, B channels respectively.
- This value is only used when the padding_mode is constant
+ i : int
+ upper pixel coordinate.
+ j : int
+ left pixel coordinate.
+ h : int
+ height of the cropped image.
+ w : int
+ width of the cropped image.
"""
- def __init__(self, padding, fill=0):
- self.padding = padding
- self.fill = fill
- def __call__(self, *args):
- return [VF.pad(img, self.padding, self.fill, padding_mode='constant') for img in args]
+ pass
+
-class AutoLevel16to8:
+class SingleAutoLevel16to8:
"""Converts a 16-bit image to 8-bit representation using "auto-level"
+ This transform assumes that the input image is gray-scaled.
+
+ To auto-level, we calculate the maximum and the minimum of the image, and
+ consider such a range should be mapped to the [0,255] range of the
+ destination image.
+
+ """
+
+ def __call__(self, img):
+ imin, imax = img.getextrema()
+ irange = imax - imin
+ return PIL.Image.fromarray(
+ numpy.round(
+ 255.0 * (numpy.array(img).astype(float) - imin) / irange
+ ).astype("uint8"),
+ ).convert("L")
+
+
+class AutoLevel16to8(TupleMixin, SingleAutoLevel16to8):
+ """Converts multiple 16-bit images to 8-bit representations using "auto-level"
+
This transform assumes that the input images are gray-scaled.
To auto-level, we calculate the maximum and the minimum of the image, and
consider such a range should be mapped to the [0,255] range of the
destination image.
"""
- def _process_one(self, img):
- return Image.fromarray(bob.core.convert(img, 'uint8', (0,255),
- img.getextrema()))
- def __call__(self, *args):
- return [self._process_one(img) for img in args]
+ pass
+
-class ToRGB:
+class SingleToRGB:
"""Converts from any input format to RGB, using an ADAPTIVE conversion.
This transform takes the input image and converts it to RGB using
- py:method:`Image.Image.convert`, with `mode='RGB'` and using all other
+ py:method:`PIL.Image.Image.convert`, with `mode='RGB'` and using all other
defaults. This may be aggressive if applied to 16-bit images without
further considerations.
"""
- def __call__(self, *args):
- return [img.convert(mode="RGB") for img in args]
-class ToTensor:
- """Converts :py:class:`PIL.Image.Image` to :py:class:`torch.Tensor` """
- def __call__(self, *args):
- return [VF.to_tensor(img) for img in args]
+ def __call__(self, img):
+ return img.convert(mode="RGB")
-# Augmentations
+class ToRGB(TupleMixin, SingleToRGB):
+ """Converts from any input format to RGB, using an ADAPTIVE conversion.
-class RandomHFlip:
+ This transform takes the input image and converts it to RGB using
+ py:method:`PIL.Image.Image.convert`, with `mode='RGB'` and using all other
+ defaults. This may be aggressive if applied to 16-bit images without
+ further considerations.
"""
- Flips horizontally
- Attributes
- ----------
- prob : float
- probability at which imgage is flipped. Defaults to ``0.5``
- """
- def __init__(self, prob = 0.5):
- self.prob = prob
+ pass
- def __call__(self, *args):
- if random.random() < self.prob:
- return [VF.hflip(img) for img in args]
+class RandomHorizontalFlip(torchvision.transforms.RandomHorizontalFlip):
+ """Randomly flips all input images horizontally"""
+
+ def __call__(self, *args):
+ if random.random() < self.p:
+ return [
+ torchvision.transforms.functional.hflip(img) for img in args
+ ]
else:
return args
-class RandomVFlip:
- """
- Flips vertically
-
- Attributes
- ----------
- prob : float
- probability at which imgage is flipped. Defaults to ``0.5``
- """
- def __init__(self, prob = 0.5):
- self.prob = prob
+class RandomVerticalFlip(torchvision.transforms.RandomVerticalFlip):
+ """Randomly flips all input images vertically"""
def __call__(self, *args):
- if random.random() < self.prob:
- return [VF.vflip(img) for img in args]
-
+ if random.random() < self.p:
+ return [
+ torchvision.transforms.functional.vflip(img) for img in args
+ ]
else:
return args
-class RandomRotation:
- """
- Rotates by degree
-
- Attributes
- ----------
- degree_range : tuple
- range of degrees in which image and ground truth are rotated. Defaults to ``(-15, +15)``
- prob : float
- probability at which imgage is rotated. Defaults to ``0.5``
- """
- def __init__(self, degree_range = (-15, +15), prob = 0.5):
- self.prob = prob
- self.degree_range = degree_range
+class RandomRotation(torchvision.transforms.RandomRotation):
+ """Randomly rotates all input images by the same amount
- def __call__(self, *args):
- if random.random() < self.prob:
- degree = random.randint(*self.degree_range)
- return [VF.rotate(img, degree, resample = Image.BILINEAR) for img in args]
- else:
- return args
+ Unlike the current torchvision implementation, we also accept a probability
+ for applying the rotation.
-class ColorJitter(object):
- """
- Randomly change the brightness, contrast, saturation and hue
- Attributes
+ Parameters
----------
- brightness : float
- how much to jitter brightness. brightness_factor
- is chosen uniformly from ``[max(0, 1 - brightness), 1 + brightness]``.
- contrast : float
- how much to jitter contrast. contrast_factor
- is chosen uniformly from ``[max(0, 1 - contrast), 1 + contrast]``.
- saturation : float
- how much to jitter saturation. saturation_factor
- is chosen uniformly from ``[max(0, 1 - saturation), 1 + saturation]``.
- hue : float
- how much to jitter hue. hue_factor is chosen uniformly from
- ``[-hue, hue]``. Should be >=0 and <= 0.5
- prob : float
- probability at which the operation is applied
- """
- def __init__(self, brightness=0.3, contrast=0.3, saturation=0.02, hue=0.02, prob=0.5):
- self.brightness = brightness
- self.contrast = contrast
- self.saturation = saturation
- self.hue = hue
- self.prob = prob
-
- @staticmethod
- def get_params(brightness, contrast, saturation, hue):
- transforms = []
- if brightness > 0:
- brightness_factor = random.uniform(max(0, 1 - brightness), 1 + brightness)
- transforms.append(Lambda(lambda img: VF.adjust_brightness(img, brightness_factor)))
- if contrast > 0:
- contrast_factor = random.uniform(max(0, 1 - contrast), 1 + contrast)
- transforms.append(Lambda(lambda img: VF.adjust_contrast(img, contrast_factor)))
+ p : :py:class:`float`, Optional
+ probability at which the operation is applied
- if saturation > 0:
- saturation_factor = random.uniform(max(0, 1 - saturation), 1 + saturation)
- transforms.append(Lambda(lambda img: VF.adjust_saturation(img, saturation_factor)))
+ **kwargs : dict
+ passed to parent. Notice that, if not set, we use the following
+ defaults here for the underlying transform from torchvision:
- if hue > 0:
- hue_factor = random.uniform(-hue, hue)
- transforms.append(Lambda(lambda img: VF.adjust_hue(img, hue_factor)))
+ * ``degrees``: 15
+ * ``resample``: ``PIL.Image.BILINEAR``
- random.shuffle(transforms)
- transform = TorchVisionCompose(transforms)
+ """
- return transform
+ def __init__(self, p=0.5, **kwargs):
+ kwargs.setdefault("degrees", 15)
+ kwargs.setdefault("resample", PIL.Image.BILINEAR)
+ super(RandomRotation, self).__init__(**kwargs)
+ self.p = p
def __call__(self, *args):
- if random.random() < self.prob:
- transform = self.get_params(self.brightness, self.contrast,
- self.saturation, self.hue)
- trans_img = transform(args[0])
- return [trans_img, *args[1:]]
+ # applies **the same** rotation to all inputs (data and ground-truth)
+ if random.random() < self.p:
+ angle = self.get_params(self.degrees)
+ return [
+ torchvision.transforms.functional.rotate(
+ img, angle, self.resample, self.expand, self.center
+ )
+ for img in args
+ ]
else:
return args
+ def __repr__(self):
+ retval = super(RandomRotation, self).__repr__()
+ return retval.replace("(", f"(p={self.p},", 1)
-class RandomResizedCrop:
- """Crop to random size and aspect ratio.
- A crop of random size of the original size and a random aspect ratio of
- the original aspect ratio is made. This crop is finally resized to
- given size. This is popularly used to train the Inception networks.
- Attributes
- ----------
- size : int
- expected output size of each edge
- scale : tuple
- range of size of the origin size cropped. Defaults to ``(0.08, 1.0)``
- ratio : tuple
- range of aspect ratio of the origin aspect ratio cropped. Defaults to ``(3. / 4., 4. / 3.)``
- interpolation :
- Defaults to ``PIL.Image.BILINEAR``
- prob : float
- probability at which the operation is applied. Defaults to ``0.5``
- """
+class ColorJitter(torchvision.transforms.ColorJitter):
+ """Randomly applies a color jitter transformation on the **first** image
- def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), interpolation=Image.BILINEAR, prob = 0.5):
- if isinstance(size, tuple):
- self.size = size
- else:
- self.size = (size, size)
- if (scale[0] > scale[1]) or (ratio[0] > ratio[1]):
- warnings.warn("range should be of kind (min, max)")
-
- self.interpolation = interpolation
- self.scale = scale
- self.ratio = ratio
- self.prob = prob
-
- @staticmethod
- def get_params(img, scale, ratio):
- area = img.size[0] * img.size[1]
-
- for attempt in range(10):
- target_area = random.uniform(*scale) * area
- log_ratio = (math.log(ratio[0]), math.log(ratio[1]))
- aspect_ratio = math.exp(random.uniform(*log_ratio))
-
- w = int(round(math.sqrt(target_area * aspect_ratio)))
- h = int(round(math.sqrt(target_area / aspect_ratio)))
-
- if w <= img.size[0] and h <= img.size[1]:
- i = random.randint(0, img.size[1] - h)
- j = random.randint(0, img.size[0] - w)
- return i, j, h, w
-
- # Fallback to central crop
- in_ratio = img.size[0] / img.size[1]
- if (in_ratio < min(ratio)):
- w = img.size[0]
- h = w / min(ratio)
- elif (in_ratio > max(ratio)):
- h = img.size[1]
- w = h * max(ratio)
- else: # whole image
- w = img.size[0]
- h = img.size[1]
- i = (img.size[1] - h) // 2
- j = (img.size[0] - w) // 2
- return i, j, h, w
+ Notice this transform extension, unlike others in this module, only affects
+ the first image passed as input argument. Unlike the current torchvision
+ implementation, we also accept a probability for applying the jitter.
- def __call__(self, *args):
- if random.random() < self.prob:
- imgs = []
- for img in args:
- i, j, h, w = self.get_params(img, self.scale, self.ratio)
- img = VF.resized_crop(img, i, j, h, w, self.size, self.interpolation)
- imgs.append(img)
- return imgs
- else:
- return args
- def __repr__(self):
- interpolate_str = _pil_interpolation_to_str[self.interpolation]
- format_string = self.__class__.__name__ + '(size={0}'.format(self.size)
- format_string += ', scale={0}'.format(tuple(round(s, 4) for s in self.scale))
- format_string += ', ratio={0}'.format(tuple(round(r, 4) for r in self.ratio))
- format_string += ', interpolation={0})'.format(interpolate_str)
- return format_string
+ Parameters
+ ----------
+ p : :py:class:`float`, Optional
+ probability at which the operation is applied
-class Resize:
- """Resize to given size.
+ **kwargs : dict
+ passed to parent. Notice that, if not set, we use the following
+ defaults here for the underlying transform from torchvision:
+
+ * ``brightness``: 0.3
+ * ``contrast``: 0.3
+ * ``saturation``: 0.02
+ * ``hue``: 0.02
- Attributes
- ----------
- size : tuple or int
- Desired output size. If size is a sequence like
- (h, w), output size will be matched to this. If size is an int,
- smaller edge of the image will be matched to this number.
- i.e, if height > width, then image will be rescaled to
- (size * height / width, size)
- interpolation : int
- Desired interpolation. Default is``PIL.Image.BILINEAR``
"""
- def __init__(self, size, interpolation=Image.BILINEAR):
- assert isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2)
- self.size = size
- self.interpolation = interpolation
+ def __init__(self, p=0.5, **kwargs):
+ kwargs.setdefault("brightness", 0.3)
+ kwargs.setdefault("contrast", 0.3)
+ kwargs.setdefault("saturation", 0.02)
+ kwargs.setdefault("hue", 0.02)
+ super(ColorJitter, self).__init__(**kwargs)
+ self.p = p
def __call__(self, *args):
- return [VF.resize(img, self.size, self.interpolation) for img in args]
+ if random.random() < self.p:
+ # applies color jitter only to the input image not ground-truth
+ return [super(ColorJitter, self).__call__(args[0]), *args[1:]]
+ else:
+ return args
def __repr__(self):
- interpolate_str = _pil_interpolation_to_str[self.interpolation]
- return self.__class__.__name__ + '(size={0}, interpolation={1})'.format(self.size, interpolate_str)
+ retval = super(ColorJitter, self).__repr__()
+ return retval.replace("(", f"(p={self.p},", 1)
diff --git a/bob/ip/binseg/data/utils.py b/bob/ip/binseg/data/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d1439a63e774d7270e76ca9b1fd3f030d6aec72
--- /dev/null
+++ b/bob/ip/binseg/data/utils.py
@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Common utilities"""
+
+import contextlib
+
+import PIL.Image
+import PIL.ImageOps
+import PIL.ImageChops
+
+import torch
+import torch.utils.data
+
+from .transforms import Compose, ToTensor
+
+
+def invert_mode1_image(img):
+ """Inverts a binary PIL image (mode == ``"1"``)"""
+
+ return PIL.ImageOps.invert(img.convert("RGB")).convert(
+ mode="1", dither=None
+ )
+
+
+def subtract_mode1_images(img1, img2):
+ """Returns a new image that represents ``img1 - img2``"""
+
+ return PIL.ImageChops.subtract(img1, img2)
+
+
+def overlayed_image(
+ img,
+ label,
+ mask=None,
+ label_color=(0, 255, 0),
+ mask_color=(0, 0, 255),
+ alpha=0.4,
+):
+ """Creates an image showing existing labels and masko
+
+ This function creates a new representation of the input image ``img``
+ overlaying a green mask for labelled objects, and a red mask for parts of
+ the image that should be ignored (negative mask). By looking at this
+ representation, it shall be possible to verify if the dataset/loader is
+ yielding images correctly.
+
+
+ Parameters
+ ----------
+
+ img : PIL.Image.Image
+ An RGB PIL image that represents the original image for analysis
+
+ label : PIL.Image.Image
+ A PIL image in any mode that represents the labelled elements in the
+ image. In case of images in mode "L" or "1", white pixels represent
+ the labelled object. Black-er pixels represent background.
+
+ mask : py:class:`PIL.Image.Image`, Optional
+ A PIL image in mode "1" that represents the mask for the image. White
+ pixels indicate where content should be used, black pixels, content to
+ be ignored.
+
+ label_color : py:class:`tuple`, Optional
+ A tuple with three integer entries indicating the RGB color to be used
+ for labels. Only used if ``label.mode`` is "1" or "L".
+
+ mask_color : py:class:`tuple`, Optional
+ A tuple with three integer entries indicating the RGB color to be used
+ for the mask-negative (black parts in the original mask).
+
+ alpha : py:class:`float`, Optional
+ A float that indicates how much of blending should be performed between
+ the label, mask and the original image.
+
+
+ Returns
+ -------
+
+ image : PIL.Image.Image
+ A new image overlaying the original image, object labels (in green) and
+ what is to be considered parts to be **masked-out** (i.e. a
+ representation of a negative of the mask).
+
+ """
+
+ # creates a representation of labels, in RGB format, with the right color
+ if label.mode in ("1", "L"):
+ label_colored = PIL.ImageOps.colorize(
+ label.convert("L"), (0, 0, 0), label_color
+ )
+ else:
+ # user has already passed an RGB version of the labels, just compose
+ label_colored = label
+
+ # blend image and label together - first blend to get vessels drawn with a
+ # slight "label_color" tone on top, then composite with original image, to
+ # avoid loosing brightness.
+ retval = PIL.Image.blend(img, label_colored, alpha)
+ if label.mode == "1":
+ composite_mask = invert_mode1_image(label)
+ else:
+ composite_mask = PIL.ImageOps.invert(label.convert("L"))
+ retval = PIL.Image.composite(img, retval, composite_mask)
+
+ # creates a representation of the mask negative with the right color
+ if mask is not None:
+ antimask_colored = PIL.ImageOps.colorize(
+ mask.convert("L"), mask_color, (0, 0, 0)
+ )
+ tmp = PIL.Image.blend(retval, antimask_colored, alpha)
+ retval = PIL.Image.composite(retval, tmp, mask)
+
+ return retval
+
+
+class SampleListDataset(torch.utils.data.Dataset):
+ """PyTorch dataset wrapper around Sample lists
+
+ A transform object can be passed that will be applied to the image, ground
+ truth and mask (if present).
+
+ It supports indexing such that dataset[i] can be used to get ith sample.
+
+
+ Attributes
+ ----------
+
+ transforms : list
+ An accessor to the list of transforms to be applied (excluding the last
+ transform, which is fixed). Notice that, after setting, a last transform
+ (:py:class:`bob.ip.binseg.data.transforms.ToTensor`) is always applied
+ - you do not need to add that.
+
+
+ Parameters
+ ----------
+
+ samples : list
+ A list of :py:class:`bob.ip.binseg.data.sample.Sample` objects
+
+ transforms : :py:class:`list`, Optional
+ a list of transformations to be applied to **both** image and
+ ground-truth data. Notice a last transform
+ (:py:class:`bob.ip.binseg.data.transforms.ToTensor`) is always applied
+ - you do not need to add that.
+
+ """
+
+ def __init__(self, samples, transforms=[]):
+
+ self._samples = samples
+ self.transforms = transforms
+
+ @property
+ def transforms(self):
+ return self._transforms.transforms[:-1]
+
+ @transforms.setter
+ def transforms(self, l):
+ self._transforms = Compose(l + [ToTensor()])
+
+ def copy(self, transforms=None):
+ """Returns a deep copy of itself, optionally resetting transforms
+
+ Parameters
+ ----------
+
+ transforms : :py:class:`list`, Optional
+ An optional list of transforms to set in the copy. If not
+ specified, use ``self.transforms``.
+ """
+
+ return SampleListDataset(self._samples, transforms or self.transforms)
+
+ def __len__(self):
+ """
+
+ Returns
+ -------
+
+ size : int
+ size of the dataset
+
+ """
+ return len(self._samples)
+
+ def __getitem__(self, key):
+ """
+
+ Parameters
+ ----------
+
+ key : int, slice
+
+ Returns
+ -------
+
+ sample : list
+ The sample data: ``[key, image[, gt[, mask]]]``
+
+ """
+
+ if isinstance(key, slice):
+ return [self[k] for k in range(*key.indices(len(self)))]
+ else: # we try it as an int
+ item = self._samples[key]
+ data = item.data # triggers data loading
+
+ retval = [data["data"]]
+ if "label" in data:
+ retval.append(data["label"])
+ if "mask" in data:
+ retval.append(data["mask"])
+
+ if self._transforms:
+ retval = self._transforms(*retval)
+
+ return [item.key] + retval
+
+
+class SSLDataset(torch.utils.data.Dataset):
+ """PyTorch dataset wrapper around labelled and unlabelled sample lists
+
+ Yields elements of the form:
+
+ .. code-block:: text
+
+ [key, image, ground-truth, [mask,] unlabelled-key, unlabelled-image]
+
+ The size of the dataset is the same as the labelled dataset.
+
+ Indexing works by selecting the right element on the labelled dataset, and
+ randomly picking another one from the unlabelled dataset
+
+ Parameters
+ ----------
+
+ labelled : :py:class:`torch.utils.data.Dataset`
+ Labelled dataset (**must** have "mask" and "label" entries for every
+ sample)
+
+ unlabelled : :py:class:`torch.utils.data.Dataset`
+ Unlabelled dataset (**may** have "mask" and "label" entries for every
+ sample, but are ignored)
+
+ """
+
+ def __init__(self, labelled, unlabelled):
+ self.labelled = labelled
+ self.unlabelled = unlabelled
+
+ def __len__(self):
+ """
+
+ Returns
+ -------
+
+ size : int
+ size of the dataset
+
+ """
+
+ return len(self.labelled)
+
+ def __getitem__(self, index):
+ """
+
+ Parameters
+ ----------
+ index : int
+ The index for the element to pick
+
+ Returns
+ -------
+
+ sample : list
+ The sample data: ``[key, image, gt, [mask, ]unlab-key, unlab-image]``
+
+ """
+
+ retval = self.labelled[index]
+ # gets one an unlabelled sample randomly to follow the labelled sample
+ unlab = self.unlabelled[torch.randint(len(self.unlabelled), ())]
+ # only interested in key and data
+ return retval + unlab[:2]
diff --git a/bob/ip/binseg/engine/__init__.py b/bob/ip/binseg/engine/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/engine/__init__.py
+++ b/bob/ip/binseg/engine/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/engine/adabound.py b/bob/ip/binseg/engine/adabound.py
index e220db5809f96d482d93059f4ee2f2bae1aec8bd..683bd76f4cf11412780bc5b78a2fa6d687707534 100644
--- a/bob/ip/binseg/engine/adabound.py
+++ b/bob/ip/binseg/engine/adabound.py
@@ -1,46 +1,70 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-"""
-https://github.com/Luolc/AdaBound/blob/master/adabound/adabound.py
-
-@inproceedings{Luo2019AdaBound,
- author = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
- title = {Adaptive Gradient Methods with Dynamic Bound of Learning Rate},
- booktitle = {Proceedings of the 7th International Conference on Learning Representations},
- month = {May},
- year = {2019},
- address = {New Orleans, Louisiana}
-}
"""
+Implementation of the `AdaBound optimizer
+<https://github.com/Luolc/AdaBound/blob/master/adabound/adabound.py>`::
+
+ @inproceedings{Luo2019AdaBound,
+ author = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ title = {Adaptive Gradient Methods with Dynamic Bound of Learning Rate},
+ booktitle = {Proceedings of the 7th International Conference on Learning Representations},
+ month = {May},
+ year = {2019},
+ address = {New Orleans, Louisiana}
+ }
+
+"""
+
import math
import torch
-from torch.optim import Optimizer
+import torch.optim
-class AdaBound(Optimizer):
- """Implements AdaBound algorithm.
- It has been proposed in `Adaptive Gradient Methods with Dynamic Bound of Learning Rate`_.
-
+class AdaBound(torch.optim.Optimizer):
+ """Implements the AdaBound algorithm.
+
Parameters
----------
- params (iterable): iterable of parameters to optimize or dicts defining
- parameter groups
- lr (float, optional): Adam learning rate (default: 1e-3)
- betas (Tuple[float, float], optional): coefficients used for computing
- running averages of gradient and its square (default: (0.9, 0.999))
- final_lr (float, optional): final (SGD) learning rate (default: 0.1)
- gamma (float, optional): convergence speed of the bound functions (default: 1e-3)
- eps (float, optional): term added to the denominator to improve
- numerical stability (default: 1e-8)
- weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
- amsbound (boolean, optional): whether to use the AMSBound variant of this algorithm
- .. Adaptive Gradient Methods with Dynamic Bound of Learning Rate:
- https://openreview.net/forum?id=Bkg3g2R9FX
+
+ params : list
+ Iterable of parameters to optimize or dicts defining parameter groups
+
+ lr : :obj:`float`, optional
+ Adam learning rate
+
+ betas : :obj:`tuple`, optional
+ Coefficients (as a 2-tuple of floats) used for computing running
+ averages of gradient and its square
+
+ final_lr : :obj:`float`, optional
+ Final (SGD) learning rate
+
+ gamma : :obj:`float`, optional
+ Convergence speed of the bound functions
+
+ eps : :obj:`float`, optional
+ Term added to the denominator to improve numerical stability
+
+ weight_decay : :obj:`float`, optional
+ Weight decay (L2 penalty)
+
+ amsbound : :obj:`bool`, optional
+ Whether to use the AMSBound variant of this algorithm
+
"""
- def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
- eps=1e-8, weight_decay=0, amsbound=False):
+ def __init__(
+ self,
+ params,
+ lr=1e-3,
+ betas=(0.9, 0.999),
+ final_lr=0.1,
+ gamma=1e-3,
+ eps=1e-8,
+ weight_decay=0,
+ amsbound=False,
+ ):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
@@ -53,60 +77,71 @@ class AdaBound(Optimizer):
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
- defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
- weight_decay=weight_decay, amsbound=amsbound)
+ defaults = dict(
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+ )
super(AdaBound, self).__init__(params, defaults)
- self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
+ self.base_lrs = list(map(lambda group: group["lr"], self.param_groups))
def __setstate__(self, state):
super(AdaBound, self).__setstate__(state)
for group in self.param_groups:
- group.setdefault('amsbound', False)
+ group.setdefault("amsbound", False)
def step(self, closure=None):
"""Performs a single optimization step.
-
+
Parameters
----------
- closure (callable, optional): A closure that reevaluates the model and returns the loss.
+
+ closure : :obj:`callable`, optional
+ A closure that reevaluates the model and returns the loss.
+
"""
loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
- for p in group['params']:
+ for p in group["params"]:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
- 'Adam does not support sparse gradients, please consider SparseAdam instead')
- amsbound = group['amsbound']
+ "Adam does not support sparse gradients, please consider SparseAdam instead"
+ )
+ amsbound = group["amsbound"]
state = self.state[p]
# State initialization
if len(state) == 0:
- state['step'] = 0
+ state["step"] = 0
# Exponential moving average of gradient values
- state['exp_avg'] = torch.zeros_like(p.data)
+ state["exp_avg"] = torch.zeros_like(p.data)
# Exponential moving average of squared gradient values
- state['exp_avg_sq'] = torch.zeros_like(p.data)
+ state["exp_avg_sq"] = torch.zeros_like(p.data)
if amsbound:
# Maintains max of all exp. moving avg. of sq. grad. values
- state['max_exp_avg_sq'] = torch.zeros_like(p.data)
+ state["max_exp_avg_sq"] = torch.zeros_like(p.data)
- exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+ exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
if amsbound:
- max_exp_avg_sq = state['max_exp_avg_sq']
- beta1, beta2 = group['betas']
+ max_exp_avg_sq = state["max_exp_avg_sq"]
+ beta1, beta2 = group["betas"]
- state['step'] += 1
+ state["step"] += 1
- if group['weight_decay'] != 0:
- grad = grad.add(group['weight_decay'], p.data)
+ if group["weight_decay"] != 0:
+ grad = grad.add(group["weight_decay"], p.data)
# Decay the first and second moment running average coefficient
exp_avg.mul_(beta1).add_(1 - beta1, grad)
@@ -115,19 +150,19 @@ class AdaBound(Optimizer):
# Maintains the maximum of all 2nd moment running avg. till now
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
# Use the max. for normalizing running avg. of gradient
- denom = max_exp_avg_sq.sqrt().add_(group['eps'])
+ denom = max_exp_avg_sq.sqrt().add_(group["eps"])
else:
- denom = exp_avg_sq.sqrt().add_(group['eps'])
+ denom = exp_avg_sq.sqrt().add_(group["eps"])
- bias_correction1 = 1 - beta1 ** state['step']
- bias_correction2 = 1 - beta2 ** state['step']
- step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1
+ bias_correction1 = 1 - beta1 ** state["step"]
+ bias_correction2 = 1 - beta2 ** state["step"]
+ step_size = group["lr"] * math.sqrt(bias_correction2) / bias_correction1
# Applies bounds on actual learning rate
# lr_scheduler cannot affect final_lr, this is a workaround to apply lr decay
- final_lr = group['final_lr'] * group['lr'] / base_lr
- lower_bound = final_lr * (1 - 1 / (group['gamma'] * state['step'] + 1))
- upper_bound = final_lr * (1 + 1 / (group['gamma'] * state['step']))
+ final_lr = group["final_lr"] * group["lr"] / base_lr
+ lower_bound = final_lr * (1 - 1 / (group["gamma"] * state["step"] + 1))
+ upper_bound = final_lr * (1 + 1 / (group["gamma"] * state["step"]))
step_size = torch.full_like(denom, step_size)
step_size.div_(denom).clamp_(lower_bound, upper_bound).mul_(exp_avg)
@@ -135,29 +170,53 @@ class AdaBound(Optimizer):
return loss
-class AdaBoundW(Optimizer):
- """Implements AdaBound algorithm with Decoupled Weight Decay (arxiv.org/abs/1711.05101)
- It has been proposed in `Adaptive Gradient Methods with Dynamic Bound of Learning Rate`_.
-
+
+class AdaBoundW(torch.optim.Optimizer):
+ """Implements AdaBound algorithm with Decoupled Weight Decay
+ (See https://arxiv.org/abs/1711.05101)
+
Parameters
----------
- params (iterable): iterable of parameters to optimize or dicts defining
- parameter groups
- lr (float, optional): Adam learning rate (default: 1e-3)
- betas (Tuple[float, float], optional): coefficients used for computing
- running averages of gradient and its square (default: (0.9, 0.999))
- final_lr (float, optional): final (SGD) learning rate (default: 0.1)
- gamma (float, optional): convergence speed of the bound functions (default: 1e-3)
- eps (float, optional): term added to the denominator to improve
- numerical stability (default: 1e-8)
- weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
- amsbound (boolean, optional): whether to use the AMSBound variant of this algorithm
- .. Adaptive Gradient Methods with Dynamic Bound of Learning Rate:
- https://openreview.net/forum?id=Bkg3g2R9FX
+
+ params : list
+ Iterable of parameters to optimize or dicts defining parameter groups
+
+ lr : :obj:`float`, optional
+ Adam learning rate
+
+ betas : :obj:`tuple`, optional
+ Coefficients (as a 2-tuple of floats) used for computing running
+ averages of gradient and its square
+
+ final_lr : :obj:`float`, optional
+ Final (SGD) learning rate
+
+ gamma : :obj:`float`, optional
+ Convergence speed of the bound functions
+
+ eps : :obj:`float`, optional
+ Term added to the denominator to improve numerical stability
+
+ weight_decay : :obj:`float`, optional
+ Weight decay (L2 penalty)
+
+ amsbound : :obj:`bool`, optional
+ Whether to use the AMSBound variant of this algorithm
+
"""
- def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
- eps=1e-8, weight_decay=0, amsbound=False):
+ def __init__(
+ self,
+ params,
+ lr=1e-3,
+ betas=(0.9, 0.999),
+ final_lr=0.1,
+ gamma=1e-3,
+ eps=1e-8,
+ weight_decay=0,
+ amsbound=False,
+ ):
+
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
@@ -170,57 +229,69 @@ class AdaBoundW(Optimizer):
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
- defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
- weight_decay=weight_decay, amsbound=amsbound)
+ defaults = dict(
+ lr=lr,
+ betas=betas,
+ final_lr=final_lr,
+ gamma=gamma,
+ eps=eps,
+ weight_decay=weight_decay,
+ amsbound=amsbound,
+ )
super(AdaBoundW, self).__init__(params, defaults)
- self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
+ self.base_lrs = list(map(lambda group: group["lr"], self.param_groups))
def __setstate__(self, state):
super(AdaBoundW, self).__setstate__(state)
for group in self.param_groups:
- group.setdefault('amsbound', False)
+ group.setdefault("amsbound", False)
def step(self, closure=None):
"""Performs a single optimization step.
-
+
Parameters
----------
- closure (callable, optional): A closure that reevaluates the model and returns the loss.
+
+ closure : :obj:`callable`, optional
+ A closure that reevaluates the model and returns the loss.
+
"""
+
loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
- for p in group['params']:
+ for p in group["params"]:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
- 'Adam does not support sparse gradients, please consider SparseAdam instead')
- amsbound = group['amsbound']
+ "Adam does not support sparse gradients, please consider SparseAdam instead"
+ )
+ amsbound = group["amsbound"]
state = self.state[p]
# State initialization
if len(state) == 0:
- state['step'] = 0
+ state["step"] = 0
# Exponential moving average of gradient values
- state['exp_avg'] = torch.zeros_like(p.data)
+ state["exp_avg"] = torch.zeros_like(p.data)
# Exponential moving average of squared gradient values
- state['exp_avg_sq'] = torch.zeros_like(p.data)
+ state["exp_avg_sq"] = torch.zeros_like(p.data)
if amsbound:
# Maintains max of all exp. moving avg. of sq. grad. values
- state['max_exp_avg_sq'] = torch.zeros_like(p.data)
+ state["max_exp_avg_sq"] = torch.zeros_like(p.data)
- exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+ exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
if amsbound:
- max_exp_avg_sq = state['max_exp_avg_sq']
- beta1, beta2 = group['betas']
+ max_exp_avg_sq = state["max_exp_avg_sq"]
+ beta1, beta2 = group["betas"]
- state['step'] += 1
+ state["step"] += 1
# Decay the first and second moment running average coefficient
exp_avg.mul_(beta1).add_(1 - beta1, grad)
@@ -229,27 +300,28 @@ class AdaBoundW(Optimizer):
# Maintains the maximum of all 2nd moment running avg. till now
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
# Use the max. for normalizing running avg. of gradient
- denom = max_exp_avg_sq.sqrt().add_(group['eps'])
+ denom = max_exp_avg_sq.sqrt().add_(group["eps"])
else:
- denom = exp_avg_sq.sqrt().add_(group['eps'])
+ denom = exp_avg_sq.sqrt().add_(group["eps"])
- bias_correction1 = 1 - beta1 ** state['step']
- bias_correction2 = 1 - beta2 ** state['step']
- step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1
+ bias_correction1 = 1 - beta1 ** state["step"]
+ bias_correction2 = 1 - beta2 ** state["step"]
+ step_size = group["lr"] * math.sqrt(bias_correction2) / bias_correction1
# Applies bounds on actual learning rate
- # lr_scheduler cannot affect final_lr, this is a workaround to apply lr decay
- final_lr = group['final_lr'] * group['lr'] / base_lr
- lower_bound = final_lr * (1 - 1 / (group['gamma'] * state['step'] + 1))
- upper_bound = final_lr * (1 + 1 / (group['gamma'] * state['step']))
+ # lr_scheduler cannot affect final_lr, this is a workaround to
+ # apply lr decay
+ final_lr = group["final_lr"] * group["lr"] / base_lr
+ lower_bound = final_lr * (1 - 1 / (group["gamma"] * state["step"] + 1))
+ upper_bound = final_lr * (1 + 1 / (group["gamma"] * state["step"]))
step_size = torch.full_like(denom, step_size)
step_size.div_(denom).clamp_(lower_bound, upper_bound).mul_(exp_avg)
- if group['weight_decay'] != 0:
- decayed_weights = torch.mul(p.data, group['weight_decay'])
+ if group["weight_decay"] != 0:
+ decayed_weights = torch.mul(p.data, group["weight_decay"])
p.data.add_(-step_size)
p.data.sub_(decayed_weights)
else:
p.data.add_(-step_size)
- return loss
\ No newline at end of file
+ return loss
diff --git a/bob/ip/binseg/engine/evaluator.py b/bob/ip/binseg/engine/evaluator.py
new file mode 100644
index 0000000000000000000000000000000000000000..07a7c86f8874dd0af50763d578a91b0a6b8a9f27
--- /dev/null
+++ b/bob/ip/binseg/engine/evaluator.py
@@ -0,0 +1,437 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Defines functionality for the evaluation of predictions"""
+
+import os
+
+import PIL
+import numpy
+import pandas
+from tqdm import tqdm
+
+import torch
+import torchvision.transforms.functional as VF
+
+import h5py
+
+from ..utils.metric import base_metrics
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def _posneg(pred, gt, threshold):
+ """Calculates true and false positives and negatives"""
+
+ gt = gt.byte() # byte tensor
+
+ # threshold
+ binary_pred = torch.gt(pred, threshold).byte()
+
+ # equals and not-equals
+ equals = torch.eq(binary_pred, gt).type(torch.uint8) # tensor
+ notequals = torch.ne(binary_pred, gt).type(torch.uint8) # tensor
+
+ # true positives
+ tp_tensor = gt * binary_pred
+
+ # false positives
+ fp_tensor = torch.eq((binary_pred + tp_tensor), 1)
+
+ # true negatives
+ tn_tensor = equals - tp_tensor
+
+ # false negatives
+ fn_tensor = notequals - fp_tensor.type(torch.uint8)
+
+ return tp_tensor, fp_tensor, tn_tensor, fn_tensor
+
+
+def _sample_metrics(pred, gt, bins):
+ """
+ Calculates metrics on one single sample and saves it to disk
+
+
+ Parameters
+ ----------
+
+ pred : torch.Tensor
+ pixel-wise predictions
+
+ gt : torch.Tensor
+ ground-truth (annotations)
+
+ bins : int
+ number of bins to use for threshold analysis. The step size is
+ calculated from this by dividing ``1.0/bins``.
+
+
+ Returns
+ -------
+
+ metrics : pandas.DataFrame
+
+ A pandas dataframe with the following columns:
+
+ * threshold: float
+ * precision: float
+ * recall: float
+ * specificity: float
+ * accuracy: float
+ * jaccard: float
+ * f1_score: float
+
+ """
+
+ step_size = 1.0 / bins
+ data = []
+
+ for index, threshold in enumerate(numpy.arange(0.0, 1.0, step_size)):
+
+ tp_tensor, fp_tensor, tn_tensor, fn_tensor = _posneg(
+ pred, gt, threshold
+ )
+
+ # calc metrics from scalars
+ tp_count = torch.sum(tp_tensor).item()
+ fp_count = torch.sum(fp_tensor).item()
+ tn_count = torch.sum(tn_tensor).item()
+ fn_count = torch.sum(fn_tensor).item()
+ (
+ precision,
+ recall,
+ specificity,
+ accuracy,
+ jaccard,
+ f1_score,
+ ) = base_metrics(tp_count, fp_count, tn_count, fn_count)
+
+ data.append(
+ [
+ index,
+ threshold,
+ precision,
+ recall,
+ specificity,
+ accuracy,
+ jaccard,
+ f1_score,
+ ]
+ )
+
+ return pandas.DataFrame(
+ data,
+ columns=(
+ "index",
+ "threshold",
+ "precision",
+ "recall",
+ "specificity",
+ "accuracy",
+ "jaccard",
+ "f1_score",
+ ),
+ )
+
+
+def _sample_analysis(
+ img,
+ pred,
+ gt,
+ threshold,
+ tp_color=(0, 255, 0), # (128,128,128) Gray
+ fp_color=(0, 0, 255), # (70, 240, 240) Cyan
+ fn_color=(255, 0, 0), # (245, 130, 48) Orange
+ overlay=True,
+):
+ """Visualizes true positives, false positives and false negatives
+
+
+ Parameters
+ ----------
+
+ img : torch.Tensor
+ original image
+
+ pred : torch.Tensor
+ pixel-wise predictions
+
+ gt : torch.Tensor
+ ground-truth (annotations)
+
+ threshold : float
+ The threshold to be used while analyzing this image's probability map
+
+ tp_color : tuple
+ RGB value for true positives
+
+ fp_color : tuple
+ RGB value for false positives
+
+ fn_color : tuple
+ RGB value for false negatives
+
+ overlay : :py:class:`bool`, Optional
+ If set to ``True`` (which is the default), then overlay annotations on
+ top of the image. Otherwise, represent data on a black canvas.
+
+
+ Returns
+ -------
+
+ figure : PIL.Image.Image
+
+ A PIL image that contains the overlayed analysis of true-positives
+ (TP), false-positives (FP) and false negatives (FN).
+
+ """
+
+ tp_tensor, fp_tensor, tn_tensor, fn_tensor = _posneg(pred, gt, threshold)
+
+ # change to PIL representation
+ tp_pil = VF.to_pil_image(tp_tensor.float())
+ tp_pil_colored = PIL.ImageOps.colorize(tp_pil, (0, 0, 0), tp_color)
+
+ fp_pil = VF.to_pil_image(fp_tensor.float())
+ fp_pil_colored = PIL.ImageOps.colorize(fp_pil, (0, 0, 0), fp_color)
+
+ fn_pil = VF.to_pil_image(fn_tensor.float())
+ fn_pil_colored = PIL.ImageOps.colorize(fn_pil, (0, 0, 0), fn_color)
+
+ tp_pil_colored.paste(fp_pil_colored, mask=fp_pil)
+ tp_pil_colored.paste(fn_pil_colored, mask=fn_pil)
+
+ if overlay:
+ img = VF.to_pil_image(img) # PIL Image
+ # using blend here, to fade original image being overlayed, or
+ # its brightness may obfuscate colors from the vessel map
+ tp_pil_colored = PIL.Image.blend(img, tp_pil_colored, 0.5)
+
+ return tp_pil_colored
+
+
+def run(
+ dataset,
+ name,
+ predictions_folder,
+ output_folder=None,
+ overlayed_folder=None,
+ threshold=None,
+):
+ """
+ Runs inference and calculates metrics
+
+
+ Parameters
+ ---------
+
+ dataset : py:class:`torch.utils.data.Dataset`
+ a dataset to iterate on
+
+ name : str
+ the local name of this dataset (e.g. ``train``, or ``test``), to be
+ used when saving metrics files.
+
+ predictions_folder : str
+ folder where predictions for the dataset images has been previously
+ stored
+
+ output_folder : :py:class:`str`, Optional
+ folder where to store results. If not provided, then do not store any
+ analysis (useful for quickly calculating overlay thresholds)
+
+ overlayed_folder : :py:class:`str`, Optional
+ if not ``None``, then it should be the name of a folder where to store
+ overlayed versions of the images and ground-truths
+
+ threshold : :py:class:`float`, Optional
+ if ``overlayed_folder``, then this should be threshold (floating point)
+ to apply to prediction maps to decide on positives and negatives for
+ overlaying analysis (graphical output). This number should come from
+ the training set or a separate validation set. Using a test set value
+ may bias your analysis. This number is also used to print the a priori
+ F1-score on the evaluated set.
+
+
+ Returns
+ -------
+
+ threshold : float
+ Threshold to achieve the highest possible F1-score for this dataset
+
+ """
+
+ # Collect overall metrics
+ bins = 100 # number of thresholds to analyse for
+ data = {}
+
+ for sample in tqdm(dataset):
+ stem = sample[0]
+ image = sample[1]
+ gt = sample[2]
+ pred_fullpath = os.path.join(predictions_folder, stem + ".hdf5")
+ with h5py.File(pred_fullpath, "r") as f:
+ pred = f["array"][:]
+ pred = torch.from_numpy(pred)
+ if stem in data:
+ raise RuntimeError(
+ f"{stem} entry already exists in data. Cannot overwrite."
+ )
+ data[stem] = _sample_metrics(pred, gt, bins)
+
+ if overlayed_folder is not None:
+ overlay_image = _sample_analysis(
+ image, pred, gt, threshold=threshold, overlay=True
+ )
+ fullpath = os.path.join(overlayed_folder, f"{stem}.png")
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ overlay_image.save(fullpath)
+
+ # Merges all dataframes together
+ df_metrics = pandas.concat(data.values())
+
+ # Report and Averages
+ avg_metrics = df_metrics.groupby("index").mean()
+ std_metrics = df_metrics.groupby("index").std()
+
+ # Uncomment below for F1-score calculation based on average precision and
+ # metrics instead of F1-scores of individual images. This method is in line
+ # with Maninis et. al. (2016)
+ #
+ # avg_metrics["f1_score"] = \
+ # (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
+ # (avg_metrics["precision"]+avg_metrics["recall"])
+
+ avg_metrics["std_pr"] = std_metrics["precision"]
+ avg_metrics["pr_upper"] = avg_metrics["precision"] + std_metrics["precision"]
+ avg_metrics["pr_lower"] = avg_metrics["precision"] - std_metrics["precision"]
+ avg_metrics["std_re"] = std_metrics["recall"]
+ avg_metrics["re_upper"] = avg_metrics["recall"] + std_metrics["recall"]
+ avg_metrics["re_lower"] = avg_metrics["recall"] - std_metrics["recall"]
+ avg_metrics["std_f1"] = std_metrics["f1_score"]
+
+ maxf1 = avg_metrics["f1_score"].max()
+ maxf1_index = avg_metrics["f1_score"].idxmax()
+ maxf1_threshold = avg_metrics["threshold"][maxf1_index]
+
+ logger.info(
+ f"Maximum F1-score of {maxf1:.5f}, achieved at "
+ f"threshold {maxf1_threshold:.3f} (chosen *a posteriori*)"
+ )
+
+ if threshold is not None:
+
+ # get the closest possible threshold we have
+ index = int(round(bins * threshold))
+ f1_a_priori = avg_metrics["f1_score"][index]
+ actual_threshold = avg_metrics["threshold"][index]
+
+ logger.info(
+ f"F1-score of {f1_a_priori:.5f}, at threshold "
+ f"{actual_threshold:.3f} (chosen *a priori*)"
+ )
+
+ if output_folder is not None:
+ logger.info(f"Output folder: {output_folder}")
+ os.makedirs(output_folder, exist_ok=True)
+ metrics_path = os.path.join(output_folder, f"{name}.csv")
+ logger.info(
+ f"Saving averages over all input images at {metrics_path}..."
+ )
+ avg_metrics.to_csv(metrics_path)
+
+ return maxf1_threshold
+
+
+def compare_annotators(baseline, other, name, output_folder,
+ overlayed_folder=None):
+ """
+ Compares annotations on the **same** dataset
+
+
+ Parameters
+ ---------
+
+ baseline : py:class:`torch.utils.data.Dataset`
+ a dataset to iterate on, containing the baseline annotations
+
+ other : py:class:`torch.utils.data.Dataset`
+ a second dataset, with the same samples as ``baseline``, but annotated
+ by a different annotator than in the first dataset.
+
+ name : str
+ the local name of this dataset (e.g. ``train-second-annotator``, or
+ ``test-second-annotator``), to be used when saving metrics files.
+
+ output_folder : str
+ folder where to store results
+
+ overlayed_folder : :py:class:`str`, Optional
+ if not ``None``, then it should be the name of a folder where to store
+ overlayed versions of the images and ground-truths
+
+ """
+
+ logger.info(f"Output folder: {output_folder}")
+ os.makedirs(output_folder, exist_ok=True)
+
+ # Collect overall metrics
+ data = {}
+
+ for baseline_sample, other_sample in tqdm(
+ list(zip(baseline, other)), desc="samples", leave=False, disable=None,
+ ):
+ stem = baseline_sample[0]
+ image = baseline_sample[1]
+ gt = baseline_sample[2]
+ pred = other_sample[2] # works as a prediction
+ if stem in data:
+ raise RuntimeError(
+ f"{stem} entry already exists in data. " f"Cannot overwrite."
+ )
+ data[stem] = _sample_metrics(pred, gt, 2)
+
+ if overlayed_folder is not None:
+ overlay_image = _sample_analysis(
+ image, pred, gt, threshold=0.5, overlay=True
+ )
+ fullpath = os.path.join(overlayed_folder, "second-annotator",
+ f"{stem}.png")
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ overlay_image.save(fullpath)
+
+ # Merges all dataframes together
+ df_metrics = pandas.concat(data.values())
+ df_metrics.drop(0, inplace=True)
+
+ # Report and Averages
+ avg_metrics = df_metrics.groupby("index").mean()
+ std_metrics = df_metrics.groupby("index").std()
+
+ # Uncomment below for F1-score calculation based on average precision and
+ # {name} instead of F1-scores of individual images. This method is in line
+ # with Maninis et. al. (2016)
+ #
+ # avg_metrics["f1_score"] = \
+ # (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
+ # (avg_metrics["precision"]+avg_metrics["recall"])
+
+ avg_metrics["std_pr"] = std_metrics["precision"]
+ avg_metrics["pr_upper"] = avg_metrics["precision"] + std_metrics["precision"]
+ avg_metrics["pr_lower"] = avg_metrics["precision"] - std_metrics["precision"]
+ avg_metrics["std_re"] = std_metrics["recall"]
+ avg_metrics["re_upper"] = avg_metrics["recall"] + std_metrics["recall"]
+ avg_metrics["re_lower"] = avg_metrics["recall"] - std_metrics["recall"]
+ avg_metrics["std_f1"] = std_metrics["f1_score"]
+
+ metrics_path = os.path.join(output_folder, "second-annotator", f"{name}.csv")
+ os.makedirs(os.path.dirname(metrics_path), exist_ok=True)
+ logger.info(f"Saving averages over all input images at {metrics_path}...")
+ avg_metrics.to_csv(metrics_path)
+
+ maxf1 = avg_metrics["f1_score"].max()
+ logger.info(f"F1-score of {maxf1:.5f} (second annotator; threshold=0.5)")
diff --git a/bob/ip/binseg/engine/inferencer.py b/bob/ip/binseg/engine/inferencer.py
deleted file mode 100644
index e76153c045fc69cabffa1e5ea7d70efacbd510a6..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/engine/inferencer.py
+++ /dev/null
@@ -1,280 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import logging
-import time
-import datetime
-import numpy as np
-import torch
-import pandas as pd
-import torchvision.transforms.functional as VF
-from tqdm import tqdm
-
-import bob.io.base
-
-from bob.ip.binseg.utils.metric import SmoothedValue, base_metrics
-from bob.ip.binseg.utils.plot import precision_recall_f1iso_confintval
-from bob.ip.binseg.utils.summary import summary
-
-
-
-def batch_metrics(predictions, ground_truths, names, output_folder, logger):
- """
- Calculates metrics on the batch and saves it to disc
-
- Parameters
- ----------
- predictions : :py:class:`torch.Tensor`
- tensor with pixel-wise probabilities
- ground_truths : :py:class:`torch.Tensor`
- tensor with binary ground-truth
- names : list
- list of file names
- output_folder : str
- output path
- logger : :py:class:`logging.Logger`
- python logger
-
- Returns
- -------
- list
- list containing batch metrics: ``[name, threshold, precision, recall, specificity, accuracy, jaccard, f1_score]``
- """
- step_size = 0.01
- batch_metrics = []
-
- for j in range(predictions.size()[0]):
- # ground truth byte
- gts = ground_truths[j].byte()
-
- file_name = "{}.csv".format(names[j])
- logger.info("saving {}".format(file_name))
-
- with open (os.path.join(output_folder,file_name), "w+") as outfile:
-
- outfile.write("threshold, precision, recall, specificity, accuracy, jaccard, f1_score\n")
-
- for threshold in np.arange(0.0,1.0,step_size):
- # threshold
- binary_pred = torch.gt(predictions[j], threshold).byte()
-
- # equals and not-equals
- equals = torch.eq(binary_pred, gts).type(torch.uint8) # tensor
- notequals = torch.ne(binary_pred, gts).type(torch.uint8) # tensor
-
- # true positives
- tp_tensor = (gts * binary_pred ) # tensor
- tp_count = torch.sum(tp_tensor).item() # scalar
-
- # false positives
- fp_tensor = torch.eq((binary_pred + tp_tensor), 1)
- fp_count = torch.sum(fp_tensor).item()
-
- # true negatives
- tn_tensor = equals - tp_tensor
- tn_count = torch.sum(tn_tensor).item()
-
- # false negatives
- fn_tensor = notequals - fp_tensor.type(torch.uint8)
- fn_count = torch.sum(fn_tensor).item()
-
- # calc metrics
- metrics = base_metrics(tp_count, fp_count, tn_count, fn_count)
-
- # write to disk
- outfile.write("{:.2f},{:.5f},{:.5f},{:.5f},{:.5f},{:.5f},{:.5f} \n".format(threshold, *metrics))
-
- batch_metrics.append([names[j],threshold, *metrics ])
-
-
- return batch_metrics
-
-
-def save_probability_images(predictions, names, output_folder, logger):
- """
- Saves probability maps as image in the same format as the test image
-
- Parameters
- ----------
- predictions : :py:class:`torch.Tensor`
- tensor with pixel-wise probabilities
- names : list
- list of file names
- output_folder : str
- output path
- logger : :py:class:`logging.Logger`
- python logger
- """
- images_subfolder = os.path.join(output_folder,'images')
- for j in range(predictions.size()[0]):
- img = VF.to_pil_image(predictions.cpu().data[j])
- filename = '{}.png'.format(names[j].split(".")[0])
- fullpath = os.path.join(images_subfolder, filename)
- logger.info("saving {}".format(fullpath))
- fulldir = os.path.dirname(fullpath)
- if not os.path.exists(fulldir): os.makedirs(fulldir)
- img.save(fullpath)
-
-def save_hdf(predictions, names, output_folder, logger):
- """
- Saves probability maps as image in the same format as the test image
-
- Parameters
- ----------
- predictions : :py:class:`torch.Tensor`
- tensor with pixel-wise probabilities
- names : list
- list of file names
- output_folder : str
- output path
- logger : :py:class:`logging.Logger`
- python logger
- """
- hdf5_subfolder = os.path.join(output_folder,'hdf5')
- if not os.path.exists(hdf5_subfolder): os.makedirs(hdf5_subfolder)
- for j in range(predictions.size()[0]):
- img = predictions.cpu().data[j].squeeze(0).numpy()
- filename = '{}.hdf5'.format(names[j].split(".")[0])
- fullpath = os.path.join(hdf5_subfolder, filename)
- logger.info("saving {}".format(filename))
- fulldir = os.path.dirname(fullpath)
- if not os.path.exists(fulldir): os.makedirs(fulldir)
- bob.io.base.save(img, fullpath)
-
-def do_inference(
- model,
- data_loader,
- device,
- output_folder = None
-):
-
- """
- Run inference and calculate metrics
-
- Parameters
- ---------
- model : :py:class:`torch.nn.Module`
- neural network model (e.g. DRIU, HED, UNet)
- data_loader : py:class:`torch.torch.utils.data.DataLoader`
- device : str
- device to use ``'cpu'`` or ``'cuda'``
- output_folder : str
- """
- logger = logging.getLogger("bob.ip.binseg.engine.inference")
- logger.info("Start evaluation")
- logger.info("Output folder: {}, Device: {}".format(output_folder, device))
- results_subfolder = os.path.join(output_folder,'results')
- os.makedirs(results_subfolder,exist_ok=True)
-
- model.eval().to(device)
- # Sigmoid for probabilities
- sigmoid = torch.nn.Sigmoid()
-
- # Setup timers
- start_total_time = time.time()
- times = []
-
- # Collect overall metrics
- metrics = []
-
- for samples in tqdm(data_loader):
- names = samples[0]
- images = samples[1].to(device)
- ground_truths = samples[2].to(device)
- with torch.no_grad():
- start_time = time.perf_counter()
-
- outputs = model(images)
-
- # necessary check for hed architecture that uses several outputs
- # for loss calculation instead of just the last concatfuse block
- if isinstance(outputs,list):
- outputs = outputs[-1]
-
- probabilities = sigmoid(outputs)
-
- batch_time = time.perf_counter() - start_time
- times.append(batch_time)
- logger.info("Batch time: {:.5f} s".format(batch_time))
-
- b_metrics = batch_metrics(probabilities, ground_truths, names,results_subfolder, logger)
- metrics.extend(b_metrics)
-
- # Create probability images
- save_probability_images(probabilities, names, output_folder, logger)
- # save hdf5
- save_hdf(probabilities, names, output_folder, logger)
-
- # DataFrame
- df_metrics = pd.DataFrame(metrics,columns= \
- ["name",
- "threshold",
- "precision",
- "recall",
- "specificity",
- "accuracy",
- "jaccard",
- "f1_score"])
-
- # Report and Averages
- metrics_file = "Metrics.csv".format(model.name)
- metrics_path = os.path.join(results_subfolder, metrics_file)
- logger.info("Saving average over all input images: {}".format(metrics_file))
-
- avg_metrics = df_metrics.groupby('threshold').mean()
- std_metrics = df_metrics.groupby('threshold').std()
-
- # Uncomment below for F1-score calculation based on average precision and metrics instead of
- # F1-scores of individual images. This method is in line with Maninis et. al. (2016)
- #avg_metrics["f1_score"] = (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
- # (avg_metrics["precision"]+avg_metrics["recall"])
-
- avg_metrics["std_pr"] = std_metrics["precision"]
- avg_metrics["pr_upper"] = avg_metrics['precision'] + avg_metrics["std_pr"]
- avg_metrics["pr_lower"] = avg_metrics['precision'] - avg_metrics["std_pr"]
- avg_metrics["std_re"] = std_metrics["recall"]
- avg_metrics["re_upper"] = avg_metrics['recall'] + avg_metrics["std_re"]
- avg_metrics["re_lower"] = avg_metrics['recall'] - avg_metrics["std_re"]
- avg_metrics["std_f1"] = std_metrics["f1_score"]
-
- avg_metrics.to_csv(metrics_path)
- maxf1 = avg_metrics['f1_score'].max()
- optimal_f1_threshold = avg_metrics['f1_score'].idxmax()
-
- logger.info("Highest F1-score of {:.5f}, achieved at threshold {}".format(maxf1, optimal_f1_threshold))
-
- # Plotting
- np_avg_metrics = avg_metrics.to_numpy().T
- fig_name = "precision_recall.pdf"
- logger.info("saving {}".format(fig_name))
- fig = precision_recall_f1iso_confintval([np_avg_metrics[0]],[np_avg_metrics[1]],[np_avg_metrics[7]],[np_avg_metrics[8]],[np_avg_metrics[10]],[np_avg_metrics[11]], [model.name,None], title=output_folder)
- fig_filename = os.path.join(results_subfolder, fig_name)
- fig.savefig(fig_filename)
-
- # Report times
- total_inference_time = str(datetime.timedelta(seconds=int(sum(times))))
- average_batch_inference_time = np.mean(times)
- total_evalution_time = str(datetime.timedelta(seconds=int(time.time() - start_total_time )))
-
- logger.info("Average batch inference time: {:.5f}s".format(average_batch_inference_time))
-
- times_file = "Times.txt"
- logger.info("saving {}".format(times_file))
-
- with open (os.path.join(results_subfolder,times_file), "w+") as outfile:
- date = datetime.datetime.now()
- outfile.write("Date: {} \n".format(date.strftime("%Y-%m-%d %H:%M:%S")))
- outfile.write("Total evaluation run-time: {} \n".format(total_evalution_time))
- outfile.write("Average batch inference time: {} \n".format(average_batch_inference_time))
- outfile.write("Total inference time: {} \n".format(total_inference_time))
-
- # Save model summary
- summary_file = 'ModelSummary.txt'
- logger.info("saving {}".format(summary_file))
-
- with open (os.path.join(results_subfolder,summary_file), "w+") as outfile:
- summary(model,outfile)
-
-
-
diff --git a/bob/ip/binseg/engine/predicter.py b/bob/ip/binseg/engine/predicter.py
deleted file mode 100644
index ebd09ac5e84d4f9a81a20f72c3919f42071fb73d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/engine/predicter.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import logging
-import time
-import datetime
-import numpy as np
-import torch
-import torchvision.transforms.functional as VF
-from tqdm import tqdm
-
-from bob.ip.binseg.utils.summary import summary
-from bob.ip.binseg.engine.inferencer import save_probability_images
-from bob.ip.binseg.engine.inferencer import save_hdf
-
-
-def do_predict(
- model,
- data_loader,
- device,
- output_folder = None
-):
-
- """
- Run inference and calculate metrics
-
- Parameters
- ---------
- model : :py:class:`torch.nn.Module`
- neural network model (e.g. DRIU, HED, UNet)
- data_loader : py:class:`torch.torch.utils.data.DataLoader`
- device : str
- device to use ``'cpu'`` or ``'cuda'``
- output_folder : str
- """
- logger = logging.getLogger("bob.ip.binseg.engine.inference")
- logger.info("Start evaluation")
- logger.info("Output folder: {}, Device: {}".format(output_folder, device))
- results_subfolder = os.path.join(output_folder,'results')
- os.makedirs(results_subfolder,exist_ok=True)
-
- model.eval().to(device)
- # Sigmoid for probabilities
- sigmoid = torch.nn.Sigmoid()
-
- # Setup timers
- start_total_time = time.time()
- times = []
-
- for samples in tqdm(data_loader):
- names = samples[0]
- images = samples[1].to(device)
- with torch.no_grad():
- start_time = time.perf_counter()
-
- outputs = model(images)
-
- # necessary check for hed architecture that uses several outputs
- # for loss calculation instead of just the last concatfuse block
- if isinstance(outputs,list):
- outputs = outputs[-1]
-
- probabilities = sigmoid(outputs)
-
- batch_time = time.perf_counter() - start_time
- times.append(batch_time)
- logger.info("Batch time: {:.5f} s".format(batch_time))
-
- # Create probability images
- save_probability_images(probabilities, names, output_folder, logger)
- # Save hdf5
- save_hdf(probabilities, names, output_folder, logger)
-
-
- # Report times
- total_inference_time = str(datetime.timedelta(seconds=int(sum(times))))
- average_batch_inference_time = np.mean(times)
- total_evalution_time = str(datetime.timedelta(seconds=int(time.time() - start_total_time )))
-
- logger.info("Average batch inference time: {:.5f}s".format(average_batch_inference_time))
-
- times_file = "Times.txt"
- logger.info("saving {}".format(times_file))
-
- with open (os.path.join(results_subfolder,times_file), "w+") as outfile:
- date = datetime.datetime.now()
- outfile.write("Date: {} \n".format(date.strftime("%Y-%m-%d %H:%M:%S")))
- outfile.write("Total evaluation run-time: {} \n".format(total_evalution_time))
- outfile.write("Average batch inference time: {} \n".format(average_batch_inference_time))
- outfile.write("Total inference time: {} \n".format(total_inference_time))
-
-
diff --git a/bob/ip/binseg/engine/predictor.py b/bob/ip/binseg/engine/predictor.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e4640f05b82bef935fd6a70db03c82ba9e1e98f
--- /dev/null
+++ b/bob/ip/binseg/engine/predictor.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import time
+import datetime
+
+import PIL
+import numpy
+from tqdm import tqdm
+
+import torch
+import torchvision.transforms.functional as VF
+
+import h5py
+
+from ..data.utils import overlayed_image
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def _save_hdf5(stem, prob, output_folder):
+ """
+ Saves prediction maps as image in the same format as the test image
+
+
+ Parameters
+ ----------
+ stem : str
+ the name of the file without extension on the original dataset
+
+ prob : PIL.Image.Image
+ Monochrome Image with prediction maps
+
+ output_folder : str
+ path where to store predictions
+
+ """
+
+ fullpath = os.path.join(output_folder, f"{stem}.hdf5")
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ with h5py.File(fullpath, 'w') as f:
+ data = prob.cpu().squeeze(0).numpy()
+ f.create_dataset("array", data=data, compression="gzip",
+ compression_opts=9)
+
+def _save_image(stem, extension, data, output_folder):
+ """Saves a PIL image into a file
+
+ Parameters
+ ----------
+
+ stem : str
+ the name of the file without extension on the original dataset
+
+ extension : str
+ an extension for the file to be saved (e.g. ``.png``)
+
+ data : PIL.Image.Image
+ RGB image with the original image, preloaded
+
+ output_folder : str
+ path where to store results
+
+ """
+
+ fullpath = os.path.join(output_folder, stem + extension)
+ tqdm.write(f"Saving {fullpath}...")
+ os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+ data.save(fullpath)
+
+
+def _save_overlayed_png(stem, image, prob, output_folder):
+ """Overlays prediction predictions vessel tree with original test image
+
+
+ Parameters
+ ----------
+
+ stem : str
+ the name of the file without extension on the original dataset
+
+ image : torch.Tensor
+ Tensor with RGB input image
+
+ prob : torch.Tensor
+ Tensor with 1-D prediction map
+
+ output_folder : str
+ path where to store results
+
+ """
+
+ image = VF.to_pil_image(image)
+ prob = VF.to_pil_image(prob.cpu())
+ _save_image(stem, '.png', overlayed_image(image, prob), output_folder)
+
+
+def run(model, data_loader, device, output_folder, overlayed_folder):
+ """
+ Runs inference on input data, outputs HDF5 files with predictions
+
+ Parameters
+ ---------
+ model : :py:class:`torch.nn.Module`
+ neural network model (e.g. driu, hed, unet)
+
+ data_loader : py:class:`torch.torch.utils.data.DataLoader`
+
+ device : str
+ device to use ``cpu`` or ``cuda:0``
+
+ output_folder : str
+ folder where to store output prediction maps (HDF5 files) and model
+ summary
+
+ overlayed_folder : str
+ folder where to store output images (PNG files)
+
+ """
+
+ logger.info(f"Output folder: {output_folder}")
+ os.makedirs(output_folder, exist_ok=True)
+
+ logger.info(f"Device: {device}")
+ model.eval().to(device)
+ # Sigmoid for predictions
+ sigmoid = torch.nn.Sigmoid()
+
+ # Setup timers
+ start_total_time = time.time()
+ times = []
+ len_samples = []
+
+ for samples in tqdm(
+ data_loader, desc="batches", leave=False, disable=None,
+ ):
+
+ names = samples[0]
+ images = samples[1].to(device)
+
+ with torch.no_grad():
+
+ start_time = time.perf_counter()
+ outputs = model(images)
+
+ # necessary check for HED architecture that uses several outputs
+ # for loss calculation instead of just the last concatfuse block
+ if isinstance(outputs, list):
+ outputs = outputs[-1]
+
+ predictions = sigmoid(outputs)
+
+ batch_time = time.perf_counter() - start_time
+ times.append(batch_time)
+ len_samples.append(len(images))
+
+ for stem, img, prob in zip(names, images, predictions):
+ _save_hdf5(stem, prob, output_folder)
+ if overlayed_folder is not None:
+ _save_overlayed_png(stem, img, prob, overlayed_folder)
+
+ # report operational summary
+ total_time = datetime.timedelta(seconds=int(time.time() - start_total_time))
+ logger.info(f"Total time: {total_time}")
+
+ average_batch_time = numpy.mean(times)
+ logger.info(f"Average batch time: {average_batch_time:g}s")
+
+ average_image_time = numpy.sum(numpy.array(times) * len_samples) / float(sum(len_samples))
+ logger.info(f"Average image time: {average_image_time:g}s")
diff --git a/bob/ip/binseg/engine/ssltrainer.py b/bob/ip/binseg/engine/ssltrainer.py
index 382431176fc33e4bf98e3cc38d4440c6f532c30d..2448782cc6b1b00965a4974af13f58e36a0dd0fc 100644
--- a/bob/ip/binseg/engine/ssltrainer.py
+++ b/bob/ip/binseg/engine/ssltrainer.py
@@ -1,126 +1,166 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import os
-import logging
+import os
+import csv
import time
import datetime
+import distutils.version
+
+import numpy
+import pandas
import torch
-import pandas as pd
from tqdm import tqdm
-import numpy as np
from bob.ip.binseg.utils.metric import SmoothedValue
from bob.ip.binseg.utils.plot import loss_curve
+import logging
+logger = logging.getLogger(__name__)
+
+PYTORCH_GE_110 = (distutils.version.StrictVersion(torch.__version__) >= "1.1.0")
+
+
def sharpen(x, T):
- temp = x**(1/T)
+ temp = x ** (1 / T)
return temp / temp.sum(dim=1, keepdim=True)
-def mix_up(alpha, input, target, unlabeled_input, unlabled_target):
+
+def mix_up(alpha, input, target, unlabelled_input, unlabled_target):
"""Applies mix up as described in [MIXMATCH_19].
-
+
Parameters
----------
alpha : float
+
input : :py:class:`torch.Tensor`
+
target : :py:class:`torch.Tensor`
- unlabeled_input : :py:class:`torch.Tensor`
+
+ unlabelled_input : :py:class:`torch.Tensor`
+
unlabled_target : :py:class:`torch.Tensor`
-
+
+
Returns
-------
+
list
+
"""
- # TODO:
+ # TODO:
with torch.no_grad():
- l = np.random.beta(alpha, alpha) # Eq (8)
- l = max(l, 1 - l) # Eq (9)
+ l = numpy.random.beta(alpha, alpha) # Eq (8)
+ l = max(l, 1 - l) # Eq (9)
# Shuffle and concat. Alg. 1 Line: 12
- w_inputs = torch.cat([input,unlabeled_input],0)
- w_targets = torch.cat([target,unlabled_target],0)
- idx = torch.randperm(w_inputs.size(0)) # get random index
-
- # Apply MixUp to labeled data and entries from W. Alg. 1 Line: 13
- input_mixedup = l * input + (1 - l) * w_inputs[idx[len(input):]]
- target_mixedup = l * target + (1 - l) * w_targets[idx[len(target):]]
-
- # Apply MixUp to unlabeled data and entries from W. Alg. 1 Line: 14
- unlabeled_input_mixedup = l * unlabeled_input + (1 - l) * w_inputs[idx[:len(unlabeled_input)]]
- unlabled_target_mixedup = l * unlabled_target + (1 - l) * w_targets[idx[:len(unlabled_target)]]
- return input_mixedup, target_mixedup, unlabeled_input_mixedup, unlabled_target_mixedup
+ w_inputs = torch.cat([input, unlabelled_input], 0)
+ w_targets = torch.cat([target, unlabled_target], 0)
+ idx = torch.randperm(w_inputs.size(0)) # get random index
+
+ # Apply MixUp to labelled data and entries from W. Alg. 1 Line: 13
+ input_mixedup = l * input + (1 - l) * w_inputs[idx[len(input) :]]
+ target_mixedup = l * target + (1 - l) * w_targets[idx[len(target) :]]
+
+ # Apply MixUp to unlabelled data and entries from W. Alg. 1 Line: 14
+ unlabelled_input_mixedup = (
+ l * unlabelled_input + (1 - l) * w_inputs[idx[: len(unlabelled_input)]]
+ )
+ unlabled_target_mixedup = (
+ l * unlabled_target + (1 - l) * w_targets[idx[: len(unlabled_target)]]
+ )
+ return (
+ input_mixedup,
+ target_mixedup,
+ unlabelled_input_mixedup,
+ unlabled_target_mixedup,
+ )
def square_rampup(current, rampup_length=16):
"""slowly ramp-up ``lambda_u``
-
+
Parameters
----------
+
current : int
current epoch
- rampup_length : int, optional
+
+ rampup_length : :obj:`int`, optional
how long to ramp up, by default 16
-
+
Returns
-------
- float
+
+ factor : float
ramp up factor
"""
+
if rampup_length == 0:
return 1.0
else:
- current = np.clip((current/ float(rampup_length))**2, 0.0, 1.0)
+ current = numpy.clip((current / float(rampup_length)) ** 2, 0.0, 1.0)
return float(current)
+
def linear_rampup(current, rampup_length=16):
"""slowly ramp-up ``lambda_u``
-
+
Parameters
----------
current : int
current epoch
- rampup_length : int, optional
+
+ rampup_length : :obj:`int`, optional
how long to ramp up, by default 16
-
+
Returns
-------
- float
+
+ factor: float
ramp up factor
+
"""
if rampup_length == 0:
return 1.0
else:
- current = np.clip(current / rampup_length, 0.0, 1.0)
+ current = numpy.clip(current / rampup_length, 0.0, 1.0)
return float(current)
-def guess_labels(unlabeled_images, model):
+
+def guess_labels(unlabelled_images, model):
"""
Calculate the average predictions by 2 augmentations: horizontal and vertical flips
+
Parameters
----------
- unlabeled_images : :py:class:`torch.Tensor`
- shape: ``[n,c,h,w]``
+
+ unlabelled_images : :py:class:`torch.Tensor`
+ ``[n,c,h,w]``
+
target : :py:class:`torch.Tensor`
-
+
Returns
-------
- :py:class:`torch.Tensor`
- shape: ``[n,c,h,w]``.
+
+ shape : :py:class:`torch.Tensor`
+ ``[n,c,h,w]``
+
"""
with torch.no_grad():
- guess1 = torch.sigmoid(model(unlabeled_images)).unsqueeze(0)
+ guess1 = torch.sigmoid(model(unlabelled_images)).unsqueeze(0)
# Horizontal flip and unsqueeze to work with batches (increase flip dimension by 1)
- hflip = torch.sigmoid(model(unlabeled_images.flip(2))).unsqueeze(0)
- guess2 = hflip.flip(3)
+ hflip = torch.sigmoid(model(unlabelled_images.flip(2))).unsqueeze(0)
+ guess2 = hflip.flip(3)
# Vertical flip and unsqueeze to work with batches (increase flip dimension by 1)
- vflip = torch.sigmoid(model(unlabeled_images.flip(3))).unsqueeze(0)
+ vflip = torch.sigmoid(model(unlabelled_images.flip(3))).unsqueeze(0)
guess3 = vflip.flip(4)
# Concat
- concat = torch.cat([guess1,guess2,guess3],0)
- avg_guess = torch.mean(concat,0)
+ concat = torch.cat([guess1, guess2, guess3], 0)
+ avg_guess = torch.mean(concat, 0)
return avg_guess
-def do_ssltrain(
+
+def run(
model,
data_loader,
optimizer,
@@ -131,41 +171,79 @@ def do_ssltrain(
device,
arguments,
output_folder,
- rampup_length
+ rampup_length,
):
- """
- Train model and save to disk.
-
+ """
+ Fits an FCN model using semi-supervised learning and saves it to disk.
+
Parameters
----------
- model : :py:class:`torch.nn.Module`
- Network (e.g. DRIU, HED, UNet)
+
+ model : :py:class:`torch.nn.Module`
+ Network (e.g. driu, hed, unet)
+
data_loader : :py:class:`torch.utils.data.DataLoader`
+
optimizer : :py:mod:`torch.optim`
+
criterion : :py:class:`torch.nn.modules.loss._Loss`
loss function
+
scheduler : :py:mod:`torch.optim`
learning rate scheduler
+
checkpointer : :py:class:`bob.ip.binseg.utils.checkpointer.DetectronCheckpointer`
checkpointer
+
checkpoint_period : int
- save a checkpoint every n epochs
- device : str
- device to use ``'cpu'`` or ``'cuda'``
+ save a checkpoint every ``n`` epochs. If set to ``0`` (zero), then do
+ not save intermediary checkpoints
+
+ device : str
+ device to use ``'cpu'`` or ``cuda:0``
+
arguments : dict
- start end end epochs
- output_folder : str
+ start and end epochs
+
+ output_folder : str
output path
- rampup_Length : int
+
+ rampup_length : int
rampup epochs
+
"""
- logger = logging.getLogger("bob.ip.binseg.engine.trainer")
- logger.info("Start training")
+
start_epoch = arguments["epoch"]
max_epoch = arguments["max_epoch"]
- # Logg to file
- with open (os.path.join(output_folder,"{}_trainlog.csv".format(model.name)), "a+",1) as outfile:
+ if not os.path.exists(output_folder):
+ logger.debug(f"Creating output directory '{output_folder}'...")
+ os.makedirs(output_folder)
+
+ # Log to file
+ logfile_name = os.path.join(output_folder, "trainlog.csv")
+
+ if arguments["epoch"] == 0 and os.path.exists(logfile_name):
+ logger.info(f"Truncating {logfile_name} - training is restarting...")
+ os.unlink(logfile_name)
+
+ logfile_fields = (
+ "epoch",
+ "total-time",
+ "eta",
+ "average-loss",
+ "median-loss",
+ "median-labelled-loss",
+ "median-unlabelled-loss",
+ "learning-rate",
+ "gpu-memory-megabytes",
+ )
+ with open(logfile_name, "a+", newline="") as logfile:
+ logwriter = csv.DictWriter(logfile, fieldnames=logfile_fields)
+
+ if arguments["epoch"] == 0:
+ logwriter.writeheader()
+
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
@@ -174,102 +252,97 @@ def do_ssltrain(
model.train().to(device)
# Total training timer
start_training_time = time.time()
+
for epoch in range(start_epoch, max_epoch):
- scheduler.step()
+ if not PYTORCH_GE_110: scheduler.step()
losses = SmoothedValue(len(data_loader))
- labeled_loss = SmoothedValue(len(data_loader))
- unlabeled_loss = SmoothedValue(len(data_loader))
+ labelled_loss = SmoothedValue(len(data_loader))
+ unlabelled_loss = SmoothedValue(len(data_loader))
epoch = epoch + 1
arguments["epoch"] = epoch
-
+
# Epoch time
start_epoch_time = time.time()
- for samples in tqdm(data_loader):
- # labeled
+ for samples in tqdm(data_loader, desc="batches", leave=False,
+ disable=None,):
+
+ # data forwarding on the existing network
+
+ # labelled
images = samples[1].to(device)
ground_truths = samples[2].to(device)
- unlabeled_images = samples[4].to(device)
- # labeled outputs
+ unlabelled_images = samples[4].to(device)
+ # labelled outputs
outputs = model(images)
- unlabeled_outputs = model(unlabeled_images)
- # guessed unlabeled outputs
- unlabeled_ground_truths = guess_labels(unlabeled_images, model)
- #unlabeled_ground_truths = sharpen(unlabeled_ground_truths,0.5)
- #images, ground_truths, unlabeled_images, unlabeled_ground_truths = mix_up(0.75, images, ground_truths, unlabeled_images, unlabeled_ground_truths)
- ramp_up_factor = square_rampup(epoch,rampup_length=rampup_length)
-
- loss, ll, ul = criterion(outputs, ground_truths, unlabeled_outputs, unlabeled_ground_truths, ramp_up_factor)
+ unlabelled_outputs = model(unlabelled_images)
+ # guessed unlabelled outputs
+ unlabelled_ground_truths = guess_labels(unlabelled_images, model)
+ # unlabelled_ground_truths = sharpen(unlabelled_ground_truths,0.5)
+ # images, ground_truths, unlabelled_images, unlabelled_ground_truths = mix_up(0.75, images, ground_truths, unlabelled_images, unlabelled_ground_truths)
+
+ # loss evaluation and learning (backward step)
+ ramp_up_factor = square_rampup(epoch, rampup_length=rampup_length)
+
+ loss, ll, ul = criterion(
+ outputs,
+ ground_truths,
+ unlabelled_outputs,
+ unlabelled_ground_truths,
+ ramp_up_factor,
+ )
optimizer.zero_grad()
loss.backward()
optimizer.step()
losses.update(loss)
- labeled_loss.update(ll)
- unlabeled_loss.update(ul)
- logger.debug("batch loss: {}".format(loss.item()))
+ labelled_loss.update(ll)
+ unlabelled_loss.update(ul)
+ logger.debug(f"batch loss: {loss.item()}")
+
+ if PYTORCH_GE_110: scheduler.step()
- if epoch % checkpoint_period == 0:
- checkpointer.save("model_{:03d}".format(epoch), **arguments)
+ if checkpoint_period and (epoch % checkpoint_period == 0):
+ checkpointer.save(f"model_{epoch:03d}", **arguments)
- if epoch == max_epoch:
+ if epoch >= max_epoch:
checkpointer.save("model_final", **arguments)
+ # computes ETA (estimated time-of-arrival; end of training) taking
+ # into consideration previous epoch performance
epoch_time = time.time() - start_epoch_time
-
-
eta_seconds = epoch_time * (max_epoch - epoch)
- eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
-
- outfile.write(("{epoch}, "
- "{avg_loss:.6f}, "
- "{median_loss:.6f}, "
- "{median_labeled_loss},"
- "{median_unlabeled_loss},"
- "{lr:.6f}, "
- "{memory:.0f}"
- "\n"
- ).format(
- eta=eta_string,
- epoch=epoch,
- avg_loss=losses.avg,
- median_loss=losses.median,
- median_labeled_loss = labeled_loss.median,
- median_unlabeled_loss = unlabeled_loss.median,
- lr=optimizer.param_groups[0]["lr"],
- memory = (torch.cuda.max_memory_allocated() / 1024.0 / 1024.0) if torch.cuda.is_available() else .0,
- )
- )
- logger.info(("eta: {eta}, "
- "epoch: {epoch}, "
- "avg. loss: {avg_loss:.6f}, "
- "median loss: {median_loss:.6f}, "
- "labeled loss: {median_labeled_loss}, "
- "unlabeled loss: {median_unlabeled_loss}, "
- "lr: {lr:.6f}, "
- "max mem: {memory:.0f}"
- ).format(
- eta=eta_string,
- epoch=epoch,
- avg_loss=losses.avg,
- median_loss=losses.median,
- median_labeled_loss = labeled_loss.median,
- median_unlabeled_loss = unlabeled_loss.median,
- lr=optimizer.param_groups[0]["lr"],
- memory = (torch.cuda.max_memory_allocated() / 1024.0 / 1024.0) if torch.cuda.is_available() else .0
- )
- )
+ current_time = time.time() - start_training_time
+ logdata = (
+ ("epoch", f"{epoch}"),
+ (
+ "total-time",
+ f"{datetime.timedelta(seconds=int(current_time))}",
+ ),
+ ("eta", f"{datetime.timedelta(seconds=int(eta_seconds))}"),
+ ("average-loss", f"{losses.avg:.6f}"),
+ ("median-loss", f"{losses.median:.6f}"),
+ ("median-labelled-loss", f"{labelled_loss.median:.6f}"),
+ ("median-unlabelled-loss", f"{unlabelled_loss.median:.6f}"),
+ ("learning-rate", f"{optimizer.param_groups[0]['lr']:.6f}"),
+ (
+ "gpu-memory-megabytes",
+ f"{torch.cuda.max_memory_allocated()/(1024.0*1024.0)}"
+ if torch.cuda.is_available()
+ else "0.0",
+ ),
+ )
+ logwriter.writerow(dict(k for k in logdata))
+ logger.info("|".join([f"{k}: {v}" for (k, v) in logdata]))
total_training_time = time.time() - start_training_time
- total_time_str = str(datetime.timedelta(seconds=total_training_time))
logger.info(
- "Total training time: {} ({:.4f} s / epoch)".format(
- total_time_str, total_training_time / (max_epoch)
- ))
-
- log_plot_file = os.path.join(output_folder,"{}_trainlog.pdf".format(model.name))
- logdf = pd.read_csv(os.path.join(output_folder,"{}_trainlog.csv".format(model.name)),header=None, names=["avg. loss", "median loss", "labeled loss", "unlabeled loss", "lr","max memory"])
- fig = loss_curve(logdf,output_folder)
- logger.info("saving {}".format(log_plot_file))
- fig.savefig(log_plot_file)
-
\ No newline at end of file
+ f"Total training time: {datetime.timedelta(seconds=total_training_time)} ({(total_training_time/max_epoch):.4f}s in average per epoch)"
+ )
+
+ # plots a version of the CSV trainlog into a PDF
+ logdf = pandas.read_csv(logfile_name, header=0, names=logfile_fields)
+ fig = loss_curve(logdf)
+ figurefile_name = os.path.join(output_folder, "trainlog.pdf")
+ logger.info(f"Saving {figurefile_name}")
+ fig.savefig(figurefile_name)
diff --git a/bob/ip/binseg/engine/trainer.py b/bob/ip/binseg/engine/trainer.py
index e083ec852491e82d2abee84421e1f14d20966a2a..d5591526fc149248f950e69694443335c85728a0 100644
--- a/bob/ip/binseg/engine/trainer.py
+++ b/bob/ip/binseg/engine/trainer.py
@@ -1,19 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import os
-import logging
+import os
+import csv
import time
+import shutil
import datetime
+import distutils.version
+
import torch
-import pandas as pd
from tqdm import tqdm
-from bob.ip.binseg.utils.metric import SmoothedValue
-from bob.ip.binseg.utils.plot import loss_curve
+from ..utils.metric import SmoothedValue
+from ..utils.summary import summary
+from ..utils.resources import cpu_constants, gpu_constants, cpu_log, gpu_log
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+PYTORCH_GE_110 = distutils.version.StrictVersion(torch.__version__) >= "1.1.0"
-def do_train(
+def run(
model,
data_loader,
optimizer,
@@ -23,40 +32,111 @@ def do_train(
checkpoint_period,
device,
arguments,
- output_folder
+ output_folder,
):
- """
- Train model and save to disk.
-
+ """
+ Fits an FCN model using supervised learning and save it to disk.
+
+ This method supports periodic checkpointing and the output of a
+ CSV-formatted log with the evolution of some figures during training.
+
+
Parameters
----------
- model : :py:class:`torch.nn.Module`
+
+ model : :py:class:`torch.nn.Module`
Network (e.g. DRIU, HED, UNet)
+
data_loader : :py:class:`torch.utils.data.DataLoader`
+
optimizer : :py:mod:`torch.optim`
+
criterion : :py:class:`torch.nn.modules.loss._Loss`
loss function
+
scheduler : :py:mod:`torch.optim`
learning rate scheduler
+
checkpointer : :py:class:`bob.ip.binseg.utils.checkpointer.DetectronCheckpointer`
- checkpointer
+ checkpointer implementation
+
checkpoint_period : int
- save a checkpoint every n epochs
- device : str
- device to use ``'cpu'`` or ``'cuda'``
+ save a checkpoint every ``n`` epochs. If set to ``0`` (zero), then do
+ not save intermediary checkpoints
+
+ device : str
+ device to use ``'cpu'`` or ``cuda:0``
+
arguments : dict
- start end end epochs
- output_folder : str
+ start and end epochs
+
+ output_folder : str
output path
"""
- logger = logging.getLogger("bob.ip.binseg.engine.trainer")
- logger.info("Start training")
+
start_epoch = arguments["epoch"]
max_epoch = arguments["max_epoch"]
- # Logg to file
- with open (os.path.join(output_folder,"{}_trainlog.csv".format(model.name)), "a+") as outfile:
-
+ if device != "cpu":
+ # asserts we do have a GPU
+ assert bool(gpu_constants()), (
+ f"Device set to '{device}', but cannot "
+ f"find a GPU (maybe nvidia-smi is not installed?)"
+ )
+
+ os.makedirs(output_folder, exist_ok=True)
+
+ # Save model summary
+ summary_path = os.path.join(output_folder, "model_summary.txt")
+ logger.info(f"Saving model summary at {summary_path}...")
+ with open(summary_path, "wt") as f:
+ r, n = summary(model)
+ logger.info(f"Model has {n} parameters...")
+ f.write(r)
+
+ # write static information to a CSV file
+ static_logfile_name = os.path.join(output_folder, "constants.csv")
+ if os.path.exists(static_logfile_name):
+ backup = static_logfile_name + "~"
+ if os.path.exists(backup):
+ os.unlink(backup)
+ shutil.move(static_logfile_name, backup)
+ with open(static_logfile_name, "w", newline="") as f:
+ logdata = cpu_constants()
+ if device != "cpu":
+ logdata += gpu_constants()
+ logdata += (("model_size", n),)
+ logwriter = csv.DictWriter(f, fieldnames=[k[0] for k in logdata])
+ logwriter.writeheader()
+ logwriter.writerow(dict(k for k in logdata))
+
+ # Log continous information to (another) file
+ logfile_name = os.path.join(output_folder, "trainlog.csv")
+
+ if arguments["epoch"] == 0 and os.path.exists(logfile_name):
+ backup = logfile_name + "~"
+ if os.path.exists(backup):
+ os.unlink(backup)
+ shutil.move(logfile_name, backup)
+
+ logfile_fields = (
+ "epoch",
+ "total_time",
+ "eta",
+ "average_loss",
+ "median_loss",
+ "learning_rate",
+ )
+ logfile_fields += tuple([k[0] for k in cpu_log()])
+ if device != "cpu":
+ logfile_fields += tuple([k[0] for k in gpu_log()])
+
+ with open(logfile_name, "a+", newline="") as logfile:
+ logwriter = csv.DictWriter(logfile, fieldnames=logfile_fields)
+
+ if arguments["epoch"] == 0:
+ logwriter.writeheader()
+
model.train().to(device)
for state in optimizer.state.values():
for k, v in state.items():
@@ -65,86 +145,78 @@ def do_train(
# Total training timer
start_training_time = time.time()
- for epoch in range(start_epoch, max_epoch):
- scheduler.step()
+ for epoch in tqdm(
+ range(start_epoch, max_epoch),
+ desc="epoch",
+ leave=False,
+ disable=None,
+ ):
+ if not PYTORCH_GE_110:
+ scheduler.step()
losses = SmoothedValue(len(data_loader))
epoch = epoch + 1
arguments["epoch"] = epoch
-
+
# Epoch time
start_epoch_time = time.time()
- for samples in tqdm(data_loader):
+ # progress bar only on interactive jobs
+ for samples in tqdm(
+ data_loader, desc="batch", leave=False, disable=None
+ ):
+ # data forwarding on the existing network
images = samples[1].to(device)
ground_truths = samples[2].to(device)
masks = None
if len(samples) == 4:
masks = samples[-1].to(device)
-
+
outputs = model(images)
-
+
+ # loss evaluation and learning (backward step)
loss = criterion(outputs, ground_truths, masks)
optimizer.zero_grad()
loss.backward()
optimizer.step()
losses.update(loss)
- logger.debug("batch loss: {}".format(loss.item()))
+ logger.debug(f"batch loss: {loss.item()}")
+
+ if PYTORCH_GE_110:
+ scheduler.step()
- if epoch % checkpoint_period == 0:
- checkpointer.save("model_{:03d}".format(epoch), **arguments)
+ if checkpoint_period and (epoch % checkpoint_period == 0):
+ checkpointer.save(f"model_{epoch:03d}", **arguments)
- if epoch == max_epoch:
+ if epoch >= max_epoch:
checkpointer.save("model_final", **arguments)
+ # computes ETA (estimated time-of-arrival; end of training) taking
+ # into consideration previous epoch performance
epoch_time = time.time() - start_epoch_time
-
-
eta_seconds = epoch_time * (max_epoch - epoch)
- eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
-
- outfile.write(("{epoch}, "
- "{avg_loss:.6f}, "
- "{median_loss:.6f}, "
- "{lr:.6f}, "
- "{memory:.0f}"
- "\n"
- ).format(
- eta=eta_string,
- epoch=epoch,
- avg_loss=losses.avg,
- median_loss=losses.median,
- lr=optimizer.param_groups[0]["lr"],
- memory = (torch.cuda.max_memory_allocated() / 1024.0 / 1024.0) if torch.cuda.is_available() else .0,
- )
- )
- logger.info(("eta: {eta}, "
- "epoch: {epoch}, "
- "avg. loss: {avg_loss:.6f}, "
- "median loss: {median_loss:.6f}, "
- "lr: {lr:.6f}, "
- "max mem: {memory:.0f}"
- ).format(
- eta=eta_string,
- epoch=epoch,
- avg_loss=losses.avg,
- median_loss=losses.median,
- lr=optimizer.param_groups[0]["lr"],
- memory = (torch.cuda.max_memory_allocated() / 1024.0 / 1024.0) if torch.cuda.is_available() else .0
- )
- )
-
+ current_time = time.time() - start_training_time
+
+ logdata = (
+ ("epoch", f"{epoch}"),
+ (
+ "total_time",
+ f"{datetime.timedelta(seconds=int(current_time))}",
+ ),
+ ("eta", f"{datetime.timedelta(seconds=int(eta_seconds))}"),
+ ("average_loss", f"{losses.avg:.6f}"),
+ ("median_loss", f"{losses.median:.6f}"),
+ ("learning_rate", f"{optimizer.param_groups[0]['lr']:.6f}"),
+ ) + cpu_log()
+ if device != 'cpu':
+ logdata += gpu_log()
+
+ logwriter.writerow(dict(k for k in logdata))
+ logfile.flush()
+ tqdm.write("|".join([f"{k}: {v}" for (k, v) in logdata[:4]]))
total_training_time = time.time() - start_training_time
- total_time_str = str(datetime.timedelta(seconds=total_training_time))
logger.info(
- "Total training time: {} ({:.4f} s / epoch)".format(
- total_time_str, total_training_time / (max_epoch)
- ))
-
- log_plot_file = os.path.join(output_folder,"{}_trainlog.pdf".format(model.name))
- logdf = pd.read_csv(os.path.join(output_folder,"{}_trainlog.csv".format(model.name)),header=None, names=["avg. loss", "median loss","lr","max memory"])
- fig = loss_curve(logdf,output_folder)
- logger.info("saving {}".format(log_plot_file))
- fig.savefig(log_plot_file)
+ f"Total training time: {datetime.timedelta(seconds=total_training_time)} ({(total_training_time/max_epoch):.4f}s in average per epoch)"
+ )
diff --git a/bob/ip/binseg/modeling/__init__.py b/bob/ip/binseg/modeling/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/modeling/__init__.py
+++ b/bob/ip/binseg/modeling/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/modeling/backbones/__init__.py b/bob/ip/binseg/modeling/backbones/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/modeling/backbones/__init__.py
+++ b/bob/ip/binseg/modeling/backbones/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/modeling/backbones/mobilenetv2.py b/bob/ip/binseg/modeling/backbones/mobilenetv2.py
index 9f1ae8f5a10ee04835efc7c40b9746f335505b31..9e6cd245a00bc5a95fef118acc26b87f136e437b 100644
--- a/bob/ip/binseg/modeling/backbones/mobilenetv2.py
+++ b/bob/ip/binseg/modeling/backbones/mobilenetv2.py
@@ -1,30 +1,30 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
-# Adopted from https://github.com/tonylins/pytorch-mobilenet-v2/ by @tonylins
+# Adopted from https://github.com/tonylins/pytorch-mobilenet-v2/ by @tonylins
# Ji Lin under Apache License 2.0
-import torch.nn as nn
+import torch.nn
import math
def conv_bn(inp, oup, stride):
- return nn.Sequential(
- nn.Conv2d(inp, oup, 3, stride, 1, bias=False),
- nn.BatchNorm2d(oup),
- nn.ReLU6(inplace=True)
+ return torch.nn.Sequential(
+ torch.nn.Conv2d(inp, oup, 3, stride, 1, bias=False),
+ torch.nn.BatchNorm2d(oup),
+ torch.nn.ReLU6(inplace=True),
)
def conv_1x1_bn(inp, oup):
- return nn.Sequential(
- nn.Conv2d(inp, oup, 1, 1, 0, bias=False),
- nn.BatchNorm2d(oup),
- nn.ReLU6(inplace=True)
+ return torch.nn.Sequential(
+ torch.nn.Conv2d(inp, oup, 1, 1, 0, bias=False),
+ torch.nn.BatchNorm2d(oup),
+ torch.nn.ReLU6(inplace=True),
)
-class InvertedResidual(nn.Module):
+class InvertedResidual(torch.nn.Module):
def __init__(self, inp, oup, stride, expand_ratio):
super(InvertedResidual, self).__init__()
self.stride = stride
@@ -34,28 +34,32 @@ class InvertedResidual(nn.Module):
self.use_res_connect = self.stride == 1 and inp == oup
if expand_ratio == 1:
- self.conv = nn.Sequential(
+ self.conv = torch.nn.Sequential(
# dw
- nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),
- nn.BatchNorm2d(hidden_dim),
- nn.ReLU6(inplace=True),
+ torch.nn.Conv2d(
+ hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False
+ ),
+ torch.nn.BatchNorm2d(hidden_dim),
+ torch.nn.ReLU6(inplace=True),
# pw-linear
- nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
- nn.BatchNorm2d(oup),
+ torch.nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
+ torch.nn.BatchNorm2d(oup),
)
else:
- self.conv = nn.Sequential(
+ self.conv = torch.nn.Sequential(
# pw
- nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False),
- nn.BatchNorm2d(hidden_dim),
- nn.ReLU6(inplace=True),
+ torch.nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False),
+ torch.nn.BatchNorm2d(hidden_dim),
+ torch.nn.ReLU6(inplace=True),
# dw
- nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),
- nn.BatchNorm2d(hidden_dim),
- nn.ReLU6(inplace=True),
+ torch.nn.Conv2d(
+ hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False
+ ),
+ torch.nn.BatchNorm2d(hidden_dim),
+ torch.nn.ReLU6(inplace=True),
# pw-linear
- nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
- nn.BatchNorm2d(oup),
+ torch.nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
+ torch.nn.BatchNorm2d(oup),
)
def forward(self, x):
@@ -65,14 +69,21 @@ class InvertedResidual(nn.Module):
return self.conv(x)
-class MobileNetV2(nn.Module):
- def __init__(self, n_class=1000, input_size=224, width_mult=1., return_features = None, m2u=True):
+class MobileNetV2(torch.nn.Module):
+ def __init__(
+ self,
+ n_class=1000,
+ input_size=224,
+ width_mult=1.0,
+ return_features=None,
+ m2u=True,
+ ):
super(MobileNetV2, self).__init__()
- self.return_features = return_features
- self.m2u = m2u
+ self.return_features = return_features
+ self.m2u = m2u
block = InvertedResidual
input_channel = 32
- last_channel = 1280
+ #last_channel = 1280
interverted_residual_setting = [
# t, c, n, s
[1, 16, 1, 1],
@@ -80,34 +91,38 @@ class MobileNetV2(nn.Module):
[6, 32, 3, 2],
[6, 64, 4, 2],
[6, 96, 3, 1],
- #[6, 160, 3, 2],
- #[6, 320, 1, 1],
+ # [6, 160, 3, 2],
+ # [6, 320, 1, 1],
]
# building first layer
assert input_size % 32 == 0
input_channel = int(input_channel * width_mult)
- #self.last_channel = int(last_channel * width_mult) if width_mult > 1.0 else last_channel
+ # self.last_channel = int(last_channel * width_mult) if width_mult > 1.0 else last_channel
self.features = [conv_bn(3, input_channel, 2)]
# building inverted residual blocks
for t, c, n, s in interverted_residual_setting:
output_channel = int(c * width_mult)
for i in range(n):
if i == 0:
- self.features.append(block(input_channel, output_channel, s, expand_ratio=t))
+ self.features.append(
+ block(input_channel, output_channel, s, expand_ratio=t)
+ )
else:
- self.features.append(block(input_channel, output_channel, 1, expand_ratio=t))
+ self.features.append(
+ block(input_channel, output_channel, 1, expand_ratio=t)
+ )
input_channel = output_channel
# building last several layers
- #self.features.append(conv_1x1_bn(input_channel, self.last_channel))
- # make it nn.Sequential
- self.features = nn.Sequential(*self.features)
+ # self.features.append(conv_1x1_bn(input_channel, self.last_channel))
+ # make it torch.nn.Sequential
+ self.features = torch.nn.Sequential(*self.features)
# building classifier
- #self.classifier = nn.Sequential(
- # nn.Dropout(0.2),
- # nn.Linear(self.last_channel, n_class),
- #)
+ # self.classifier = torch.nn.Sequential(
+ # torch.nn.Dropout(0.2),
+ # torch.nn.Linear(self.last_channel, n_class),
+ # )
self._initialize_weights()
@@ -117,7 +132,7 @@ class MobileNetV2(nn.Module):
outputs.append(x.shape[2:4])
if self.m2u:
outputs.append(x)
- for index,m in enumerate(self.features):
+ for index, m in enumerate(self.features):
x = m(x)
# extract layers
if index in self.return_features:
@@ -126,15 +141,15 @@ class MobileNetV2(nn.Module):
def _initialize_weights(self):
for m in self.modules():
- if isinstance(m, nn.Conv2d):
+ if isinstance(m, torch.nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
- m.weight.data.normal_(0, math.sqrt(2. / n))
+ m.weight.data.normal_(0, math.sqrt(2.0 / n))
if m.bias is not None:
m.bias.data.zero_()
- elif isinstance(m, nn.BatchNorm2d):
+ elif isinstance(m, torch.nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
- elif isinstance(m, nn.Linear):
+ elif isinstance(m, torch.nn.Linear):
n = m.weight.size(1)
m.weight.data.normal_(0, 0.01)
- m.bias.data.zero_()
\ No newline at end of file
+ m.bias.data.zero_()
diff --git a/bob/ip/binseg/modeling/backbones/resnet.py b/bob/ip/binseg/modeling/backbones/resnet.py
index 5881652e571c94cd0aff20082a90986feffa96db..445c4ba715cb756a0c90ad688d493e6e0d3f2535 100644
--- a/bob/ip/binseg/modeling/backbones/resnet.py
+++ b/bob/ip/binseg/modeling/backbones/resnet.py
@@ -1,28 +1,25 @@
-# Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
+# Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
# resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN : https://github.com/rgeirhos/texture-vs-shap
import torch.nn as nn
import torch.utils.model_zoo as model_zoo
-__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
- 'resnet152']
-
-
model_urls = {
- 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
- 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
- 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
- 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
- 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
- 'resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN': 'https://bitbucket.org/robert_geirhos/texture-vs-shape-pretrained-models/raw/60b770e128fffcbd8562a3ab3546c1a735432d03/resnet50_finetune_60_epochs_lr_decay_after_30_start_resnet50_train_45_epochs_combined_IN_SF-ca06340c.pth.tar',
+ "resnet18": "https://download.pytorch.org/models/resnet18-5c106cde.pth",
+ "resnet34": "https://download.pytorch.org/models/resnet34-333f7ec4.pth",
+ "resnet50": "https://download.pytorch.org/models/resnet50-19c8e357.pth",
+ "resnet101": "https://download.pytorch.org/models/resnet101-5d3b4d8f.pth",
+ "resnet152": "https://download.pytorch.org/models/resnet152-b121ed2d.pth",
+ "resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN": "https://bitbucket.org/robert_geirhos/texture-vs-shape-pretrained-models/raw/60b770e128fffcbd8562a3ab3546c1a735432d03/resnet50_finetune_60_epochs_lr_decay_after_30_start_resnet50_train_45_epochs_combined_IN_SF-ca06340c.pth.tar",
}
def conv3x3(in_planes, out_planes, stride=1):
"""3x3 convolution with padding"""
- return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
- padding=1, bias=False)
+ return nn.Conv2d(
+ in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False,
+ )
def conv1x1(in_planes, out_planes, stride=1):
@@ -101,20 +98,18 @@ class Bottleneck(nn.Module):
class ResNet(nn.Module):
-
def __init__(self, block, layers, return_features, zero_init_residual=False):
"""
Generic ResNet network with layer return.
Attributes
----------
return_features: list of length 5
- layers to return.
+ layers to return.
"""
super(ResNet, self).__init__()
self.inplanes = 64
self.return_features = return_features
- self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
- bias=False)
+ self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
@@ -123,13 +118,20 @@ class ResNet(nn.Module):
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
-
- self.features = [self.conv1, self.bn1, self.relu, self.maxpool
- ,self.layer1,self.layer2,self.layer3,self.layer4]
+ self.features = [
+ self.conv1,
+ self.bn1,
+ self.relu,
+ self.maxpool,
+ self.layer1,
+ self.layer2,
+ self.layer3,
+ self.layer4,
+ ]
for m in self.modules():
if isinstance(m, nn.Conv2d):
- nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+ nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
@@ -164,7 +166,7 @@ class ResNet(nn.Module):
outputs = []
# hw of input, needed for DRIU and HED
outputs.append(x.shape[2:4])
- for index,m in enumerate(self.features):
+ for index, m in enumerate(self.features):
x = m(x)
# extract layers
if index in self.return_features:
@@ -179,7 +181,7 @@ def resnet18(pretrained=False, **kwargs):
"""
model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
+ model.load_state_dict(model_zoo.load_url(model_urls["resnet18"]))
return model
@@ -190,7 +192,7 @@ def resnet34(pretrained=False, **kwargs):
"""
model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet34']))
+ model.load_state_dict(model_zoo.load_url(model_urls["resnet34"]))
return model
@@ -201,9 +203,10 @@ def resnet50(pretrained=False, **kwargs):
"""
model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet50']))
+ model.load_state_dict(model_zoo.load_url(model_urls["resnet50"]))
return model
+
def shaperesnet50(pretrained=False, **kwargs):
"""Constructs a ResNet-50 model, pretrained on Stylized-ImageNe and ImageNet and fine-tuned on ImageNet.
Args:
@@ -211,9 +214,14 @@ def shaperesnet50(pretrained=False, **kwargs):
"""
model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN']))
+ model.load_state_dict(
+ model_zoo.load_url(
+ model_urls["resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN"]
+ )
+ )
return model
+
def resnet101(pretrained=False, **kwargs):
"""Constructs a ResNet-101 model.
Args:
@@ -221,9 +229,10 @@ def resnet101(pretrained=False, **kwargs):
"""
model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet101']))
+ model.load_state_dict(model_zoo.load_url(model_urls["resnet101"]))
return model
+
def resnet152(pretrained=False, **kwargs):
"""Constructs a ResNet-152 model.
Args:
@@ -231,5 +240,5 @@ def resnet152(pretrained=False, **kwargs):
"""
model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['resnet152']))
- return model
\ No newline at end of file
+ model.load_state_dict(model_zoo.load_url(model_urls["resnet152"]))
+ return model
diff --git a/bob/ip/binseg/modeling/backbones/vgg.py b/bob/ip/binseg/modeling/backbones/vgg.py
index 85a375805d5182309101067e6163e651deebc44b..e3909fcbb2096f3c91a1b8e3f359dae42365f22b 100644
--- a/bob/ip/binseg/modeling/backbones/vgg.py
+++ b/bob/ip/binseg/modeling/backbones/vgg.py
@@ -1,32 +1,25 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py
+# Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py
import torch.nn as nn
import torch.utils.model_zoo as model_zoo
-__all__ = [
- 'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',
- 'vgg19_bn', 'vgg19',
-]
-
-
model_urls = {
- 'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth',
- 'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth',
- 'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth',
- 'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth',
- 'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth',
- 'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth',
- 'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth',
- 'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth',
+ "vgg11": "https://download.pytorch.org/models/vgg11-bbd30ac9.pth",
+ "vgg13": "https://download.pytorch.org/models/vgg13-c768596a.pth",
+ "vgg16": "https://download.pytorch.org/models/vgg16-397923af.pth",
+ "vgg19": "https://download.pytorch.org/models/vgg19-dcbb9e9d.pth",
+ "vgg11_bn": "https://download.pytorch.org/models/vgg11_bn-6002323d.pth",
+ "vgg13_bn": "https://download.pytorch.org/models/vgg13_bn-abd245e5.pth",
+ "vgg16_bn": "https://download.pytorch.org/models/vgg16_bn-6c64b313.pth",
+ "vgg19_bn": "https://download.pytorch.org/models/vgg19_bn-c79401a0.pth",
}
class VGG(nn.Module):
-
def __init__(self, features, return_features, init_weights=True):
super(VGG, self).__init__()
self.features = features
@@ -38,7 +31,7 @@ class VGG(nn.Module):
outputs = []
# hw of input, needed for DRIU and HED
outputs.append(x.shape[2:4])
- for index,m in enumerate(self.features):
+ for index, m in enumerate(self.features):
x = m(x)
# extract layers
if index in self.return_features:
@@ -48,7 +41,7 @@ class VGG(nn.Module):
def _initialize_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
- nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+ nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
if m.bias is not None:
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
@@ -63,7 +56,7 @@ def make_layers(cfg, batch_norm=False):
layers = []
in_channels = 3
for v in cfg:
- if v == 'M':
+ if v == "M":
layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
else:
conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
@@ -75,11 +68,52 @@ def make_layers(cfg, batch_norm=False):
return nn.Sequential(*layers)
-cfg = {
- 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
- 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
- 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
- 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
+_cfg = {
+ "A": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
+ "B": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
+ "D": [
+ 64,
+ 64,
+ "M",
+ 128,
+ 128,
+ "M",
+ 256,
+ 256,
+ 256,
+ "M",
+ 512,
+ 512,
+ 512,
+ "M",
+ 512,
+ 512,
+ 512,
+ "M",
+ ],
+ "E": [
+ 64,
+ 64,
+ "M",
+ 128,
+ 128,
+ "M",
+ 256,
+ 256,
+ 256,
+ 256,
+ "M",
+ 512,
+ 512,
+ 512,
+ 512,
+ "M",
+ 512,
+ 512,
+ 512,
+ 512,
+ "M",
+ ],
}
@@ -89,10 +123,10 @@ def vgg11(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['A']), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["A"]), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg11']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg11"]))
return model
@@ -102,10 +136,10 @@ def vgg11_bn(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['A'], batch_norm=True), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["A"], batch_norm=True), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg11_bn']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg11_bn"]))
return model
@@ -115,10 +149,10 @@ def vgg13(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['B']), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["B"]), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg13']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg13"]))
return model
@@ -128,10 +162,10 @@ def vgg13_bn(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['B'], batch_norm=True), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["B"], batch_norm=True), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg13_bn']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg13_bn"]))
return model
@@ -141,10 +175,10 @@ def vgg16(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['D']), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["D"]), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg16']),strict=False)
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg16"]), strict=False)
return model
@@ -154,10 +188,10 @@ def vgg16_bn(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['D'], batch_norm=True), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["D"], batch_norm=True), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg16_bn']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg16_bn"]))
return model
@@ -167,10 +201,10 @@ def vgg19(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['E']), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["E"]), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg19']))
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg19"]))
return model
@@ -180,8 +214,8 @@ def vgg19_bn(pretrained=False, **kwargs):
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
- kwargs['init_weights'] = False
- model = VGG(make_layers(cfg['E'], batch_norm=True), **kwargs)
+ kwargs["init_weights"] = False
+ model = VGG(make_layers(_cfg["E"], batch_norm=True), **kwargs)
if pretrained:
- model.load_state_dict(model_zoo.load_url(model_urls['vgg19_bn']))
- return model
\ No newline at end of file
+ model.load_state_dict(model_zoo.load_url(model_urls["vgg19_bn"]))
+ return model
diff --git a/bob/ip/binseg/modeling/driu.py b/bob/ip/binseg/modeling/driu.py
index 466d4eb0942dcbc5e8f7f26a4ce2ca8db52d6f38..c63dc843a7a267fd9a2f1f6bbc7dd20b4cd1a2a1 100644
--- a/bob/ip/binseg/modeling/driu.py
+++ b/bob/ip/binseg/modeling/driu.py
@@ -2,79 +2,99 @@
# -*- coding: utf-8 -*-
import torch
-from torch import nn
+import torch.nn
from collections import OrderedDict
from bob.ip.binseg.modeling.backbones.vgg import vgg16
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform,convtrans_with_kaiming_uniform, UpsampleCropBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ UpsampleCropBlock,
+)
-class ConcatFuseBlock(nn.Module):
- """
- Takes in four feature maps with 16 channels each, concatenates them
- and applies a 1x1 convolution with 1 output channel.
+
+class ConcatFuseBlock(torch.nn.Module):
+ """
+ Takes in four feature maps with 16 channels each, concatenates them
+ and applies a 1x1 convolution with 1 output channel.
"""
+
def __init__(self):
super().__init__()
- self.conv = conv_with_kaiming_uniform(4*16,1,1,1,0)
-
- def forward(self,x1,x2,x3,x4):
- x_cat = torch.cat([x1,x2,x3,x4],dim=1)
+ self.conv = conv_with_kaiming_uniform(4 * 16, 1, 1, 1, 0)
+
+ def forward(self, x1, x2, x3, x4):
+
+ x_cat = torch.cat([x1, x2, x3, x4], dim=1)
x = self.conv(x_cat)
- return x
-
-class DRIU(nn.Module):
+ return x
+
+
+class DRIU(torch.nn.Module):
"""
DRIU head module
- Based on paper by `Maninis et al. (2016)`_
+
+ Based on paper by [MANINIS-2016]_.
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None):
super(DRIU, self).__init__()
- in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8 = in_channels_list
+ (in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8,) = in_channels_list
- self.conv1_2_16 = nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
+ self.conv1_2_16 = torch.nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
# Upsample layers
self.upsample2 = UpsampleCropBlock(in_upsample2, 16, 4, 2, 0)
self.upsample4 = UpsampleCropBlock(in_upsample_4, 16, 8, 4, 0)
self.upsample8 = UpsampleCropBlock(in_upsample_8, 16, 16, 8, 0)
-
+
# Concat and Fuse
self.concatfuse = ConcatFuseBlock()
- def forward(self,x):
+ def forward(self, x):
"""
+
Parameters
----------
+
x : list
- list of tensors as returned from the backbone network.
- First element: height and width of input image.
- Remaining elements: feature maps for each feature level.
+ list of tensors as returned from the backbone network. First
+ element: height and width of input image. Remaining elements:
+ feature maps for each feature level.
Returns
-------
- :py:class:`torch.Tensor`
+
+ tensor : :py:class:`torch.Tensor`
+
"""
hw = x[0]
- conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
- upsample2 = self.upsample2(x[2], hw) # side-multi2-up
- upsample4 = self.upsample4(x[3], hw) # side-multi3-up
- upsample8 = self.upsample8(x[4], hw) # side-multi4-up
+ conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
+ upsample2 = self.upsample2(x[2], hw) # side-multi2-up
+ upsample4 = self.upsample4(x[3], hw) # side-multi3-up
+ upsample8 = self.upsample8(x[4], hw) # side-multi4-up
out = self.concatfuse(conv1_2_16, upsample2, upsample4, upsample8)
return out
+
def build_driu():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+
+ module : :py:class:`torch.nn.Module`
+
"""
- backbone = vgg16(pretrained=False, return_features = [3, 8, 14, 22])
+ backbone = vgg16(pretrained=False, return_features=[3, 8, 14, 22])
driu_head = DRIU([64, 128, 256, 512])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", driu_head)]))
- model.name = "DRIU"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", driu_head)])
+ )
+ model.name = "driu"
+ return model
diff --git a/bob/ip/binseg/modeling/driubn.py b/bob/ip/binseg/modeling/driubn.py
index 6043fcd6257b73712572002d1895fca84ca7e69f..fd834353eedd5a15e7fa017b97e517966f2a740c 100644
--- a/bob/ip/binseg/modeling/driubn.py
+++ b/bob/ip/binseg/modeling/driubn.py
@@ -2,56 +2,66 @@
# -*- coding: utf-8 -*-
import torch
-from torch import nn
+import torch.nn
from collections import OrderedDict
from bob.ip.binseg.modeling.backbones.vgg import vgg16_bn
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform,convtrans_with_kaiming_uniform, UpsampleCropBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ UpsampleCropBlock,
+)
-class ConcatFuseBlock(nn.Module):
- """
- Takes in four feature maps with 16 channels each, concatenates them
- and applies a 1x1 convolution with 1 output channel.
+
+class ConcatFuseBlock(torch.nn.Module):
+ """
+ Takes in four feature maps with 16 channels each, concatenates them
+ and applies a 1x1 convolution with 1 output channel.
"""
+
def __init__(self):
super().__init__()
- self.conv = nn.Sequential(
- conv_with_kaiming_uniform(4*16,1,1,1,0)
- ,nn.BatchNorm2d(1)
+ self.conv = torch.nn.Sequential(
+ conv_with_kaiming_uniform(4 * 16, 1, 1, 1, 0), torch.nn.BatchNorm2d(1)
)
- def forward(self,x1,x2,x3,x4):
- x_cat = torch.cat([x1,x2,x3,x4],dim=1)
+
+ def forward(self, x1, x2, x3, x4):
+ x_cat = torch.cat([x1, x2, x3, x4], dim=1)
x = self.conv(x_cat)
- return x
-
-class DRIU(nn.Module):
+ return x
+
+
+class DRIU(torch.nn.Module):
"""
DRIU head module
- Based on paper by `Maninis et al. (2016)`_
+
+ Based on paper by [MANINIS-2016]_.
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None):
super(DRIU, self).__init__()
in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8 = in_channels_list
- self.conv1_2_16 = nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
+ self.conv1_2_16 = torch.nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
# Upsample layers
self.upsample2 = UpsampleCropBlock(in_upsample2, 16, 4, 2, 0)
self.upsample4 = UpsampleCropBlock(in_upsample_4, 16, 8, 4, 0)
self.upsample8 = UpsampleCropBlock(in_upsample_8, 16, 16, 8, 0)
-
+
# Concat and Fuse
self.concatfuse = ConcatFuseBlock()
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
Returns
@@ -59,24 +69,29 @@ class DRIU(nn.Module):
:py:class:`torch.Tensor`
"""
hw = x[0]
- conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
- upsample2 = self.upsample2(x[2], hw) # side-multi2-up
- upsample4 = self.upsample4(x[3], hw) # side-multi3-up
- upsample8 = self.upsample8(x[4], hw) # side-multi4-up
+ conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
+ upsample2 = self.upsample2(x[2], hw) # side-multi2-up
+ upsample4 = self.upsample4(x[3], hw) # side-multi3-up
+ upsample8 = self.upsample8(x[4], hw) # side-multi4-up
out = self.concatfuse(conv1_2_16, upsample2, upsample4, upsample8)
return out
+
def build_driu():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+
+ module : :py:class:`torch.nn.Module`
+
"""
- backbone = vgg16_bn(pretrained=False, return_features = [5, 12, 19, 29])
+ backbone = vgg16_bn(pretrained=False, return_features=[5, 12, 19, 29])
driu_head = DRIU([64, 128, 256, 512])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", driu_head)]))
- model.name = "DRIUBN"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", driu_head)])
+ )
+ model.name = "driu-bn"
+ return model
diff --git a/bob/ip/binseg/modeling/driuod.py b/bob/ip/binseg/modeling/driuod.py
index cfa11973279ecba3b71401f5c87b661f77b34ae6..dbd26167eda41e196bf33a6c52eb9e0cf63bc59d 100644
--- a/bob/ip/binseg/modeling/driuod.py
+++ b/bob/ip/binseg/modeling/driuod.py
@@ -2,34 +2,42 @@
# -*- coding: utf-8 -*-
import torch
-from torch import nn
+import torch.nn
from collections import OrderedDict
from bob.ip.binseg.modeling.backbones.vgg import vgg16
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform,convtrans_with_kaiming_uniform, UpsampleCropBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ UpsampleCropBlock,
+)
-class ConcatFuseBlock(nn.Module):
- """
- Takes in four feature maps with 16 channels each, concatenates them
- and applies a 1x1 convolution with 1 output channel.
+
+class ConcatFuseBlock(torch.nn.Module):
+ """
+ Takes in four feature maps with 16 channels each, concatenates them
+ and applies a 1x1 convolution with 1 output channel.
"""
+
def __init__(self):
super().__init__()
- self.conv = conv_with_kaiming_uniform(4*16,1,1,1,0)
-
- def forward(self,x1,x2,x3,x4):
- x_cat = torch.cat([x1,x2,x3,x4],dim=1)
+ self.conv = conv_with_kaiming_uniform(4 * 16, 1, 1, 1, 0)
+
+ def forward(self, x1, x2, x3, x4):
+ x_cat = torch.cat([x1, x2, x3, x4], dim=1)
x = self.conv(x_cat)
- return x
-
-class DRIUOD(nn.Module):
+ return x
+
+
+class DRIUOD(torch.nn.Module):
"""
DRIU head module
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None):
super(DRIUOD, self).__init__()
in_upsample2, in_upsample_4, in_upsample_8, in_upsample_16 = in_channels_list
@@ -40,17 +48,16 @@ class DRIUOD(nn.Module):
self.upsample8 = UpsampleCropBlock(in_upsample_8, 16, 16, 8, 0)
self.upsample16 = UpsampleCropBlock(in_upsample_16, 16, 32, 16, 0)
-
# Concat and Fuse
self.concatfuse = ConcatFuseBlock()
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
Returns
@@ -59,23 +66,27 @@ class DRIUOD(nn.Module):
"""
hw = x[0]
upsample2 = self.upsample2(x[1], hw) # side-multi2-up
- upsample4 = self.upsample4(x[2], hw) # side-multi3-up
- upsample8 = self.upsample8(x[3], hw) # side-multi4-up
+ upsample4 = self.upsample4(x[2], hw) # side-multi3-up
+ upsample8 = self.upsample8(x[3], hw) # side-multi4-up
upsample16 = self.upsample16(x[4], hw) # side-multi5-up
- out = self.concatfuse(upsample2, upsample4, upsample8,upsample16)
+ out = self.concatfuse(upsample2, upsample4, upsample8, upsample16)
return out
+
def build_driuod():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+ module : :py:class:`torch.nn.Module`
+
"""
- backbone = vgg16(pretrained=False, return_features = [8, 14, 22,29])
- driu_head = DRIUOD([128, 256, 512,512])
+ backbone = vgg16(pretrained=False, return_features=[8, 14, 22, 29])
+ driu_head = DRIUOD([128, 256, 512, 512])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", driu_head)]))
- model.name = "DRIUOD"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", driu_head)])
+ )
+ model.name = "driu-od"
+ return model
diff --git a/bob/ip/binseg/modeling/driupix.py b/bob/ip/binseg/modeling/driupix.py
index 00e40932afc0c04f52925783fe3595fa22a7d098..eef95c9f9ac237c1f9235b7fc34f3e90752ccfeb 100644
--- a/bob/ip/binseg/modeling/driupix.py
+++ b/bob/ip/binseg/modeling/driupix.py
@@ -2,54 +2,66 @@
# -*- coding: utf-8 -*-
import torch
-from torch import nn
+import torch.nn
from collections import OrderedDict
from bob.ip.binseg.modeling.backbones.vgg import vgg16
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform,convtrans_with_kaiming_uniform, UpsampleCropBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ UpsampleCropBlock,
+)
-class ConcatFuseBlock(nn.Module):
- """
- Takes in four feature maps with 16 channels each, concatenates them
- and applies a 1x1 convolution with 1 output channel.
+
+class ConcatFuseBlock(torch.nn.Module):
+ """
+ Takes in four feature maps with 16 channels each, concatenates them
+ and applies a 1x1 convolution with 1 output channel.
"""
+
def __init__(self):
super().__init__()
- self.conv = conv_with_kaiming_uniform(4*16,1,1,1,0)
-
- def forward(self,x1,x2,x3,x4):
- x_cat = torch.cat([x1,x2,x3,x4],dim=1)
+ self.conv = conv_with_kaiming_uniform(4 * 16, 1, 1, 1, 0)
+
+ def forward(self, x1, x2, x3, x4):
+ x_cat = torch.cat([x1, x2, x3, x4], dim=1)
x = self.conv(x_cat)
- return x
-
-class DRIUPIX(nn.Module):
+ return x
+
+
+class DRIUPIX(torch.nn.Module):
"""
DRIUPIX head module. DRIU with pixelshuffle instead of ConvTrans2D
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None):
super(DRIUPIX, self).__init__()
in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8 = in_channels_list
- self.conv1_2_16 = nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
+ self.conv1_2_16 = torch.nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
# Upsample layers
self.upsample2 = UpsampleCropBlock(in_upsample2, 16, 4, 2, 0, pixelshuffle=True)
- self.upsample4 = UpsampleCropBlock(in_upsample_4, 16, 8, 4, 0, pixelshuffle=True)
- self.upsample8 = UpsampleCropBlock(in_upsample_8, 16, 16, 8, 0, pixelshuffle=True)
-
+ self.upsample4 = UpsampleCropBlock(
+ in_upsample_4, 16, 8, 4, 0, pixelshuffle=True
+ )
+ self.upsample8 = UpsampleCropBlock(
+ in_upsample_8, 16, 16, 8, 0, pixelshuffle=True
+ )
+
# Concat and Fuse
self.concatfuse = ConcatFuseBlock()
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
Returns
@@ -57,24 +69,28 @@ class DRIUPIX(nn.Module):
:py:class:`torch.Tensor`
"""
hw = x[0]
- conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
- upsample2 = self.upsample2(x[2], hw) # side-multi2-up
- upsample4 = self.upsample4(x[3], hw) # side-multi3-up
- upsample8 = self.upsample8(x[4], hw) # side-multi4-up
+ conv1_2_16 = self.conv1_2_16(x[1]) # conv1_2_16
+ upsample2 = self.upsample2(x[2], hw) # side-multi2-up
+ upsample4 = self.upsample4(x[3], hw) # side-multi3-up
+ upsample8 = self.upsample8(x[4], hw) # side-multi4-up
out = self.concatfuse(conv1_2_16, upsample2, upsample4, upsample8)
return out
+
def build_driupix():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+ module : :py:class:`torch.nn.Module`
+
"""
- backbone = vgg16(pretrained=False, return_features = [3, 8, 14, 22])
+ backbone = vgg16(pretrained=False, return_features=[3, 8, 14, 22])
driu_head = DRIUPIX([64, 128, 256, 512])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", driu_head)]))
- model.name = "DRIUPIX"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", driu_head)])
+ )
+ model.name = "driu-pix"
+ return model
diff --git a/bob/ip/binseg/modeling/hed.py b/bob/ip/binseg/modeling/hed.py
index fa44366e4e75dce1059a130e83f78e7884922501..db42515c78c137b93153df9f33d17e7a3a3bd7a1 100644
--- a/bob/ip/binseg/modeling/hed.py
+++ b/bob/ip/binseg/modeling/hed.py
@@ -2,82 +2,101 @@
# -*- coding: utf-8 -*-
import torch
-from torch import nn
+import torch.nn
from collections import OrderedDict
from bob.ip.binseg.modeling.backbones.vgg import vgg16
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform, convtrans_with_kaiming_uniform, UpsampleCropBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ UpsampleCropBlock,
+)
-class ConcatFuseBlock(nn.Module):
- """
- Takes in five feature maps with one channel each, concatenates thems
- and applies a 1x1 convolution with 1 output channel.
+
+class ConcatFuseBlock(torch.nn.Module):
+ """
+ Takes in five feature maps with one channel each, concatenates thems
+ and applies a 1x1 convolution with 1 output channel.
"""
+
def __init__(self):
super().__init__()
- self.conv = conv_with_kaiming_uniform(5,1,1,1,0)
-
- def forward(self,x1,x2,x3,x4,x5):
- x_cat = torch.cat([x1,x2,x3,x4,x5],dim=1)
+ self.conv = conv_with_kaiming_uniform(5, 1, 1, 1, 0)
+
+ def forward(self, x1, x2, x3, x4, x5):
+ x_cat = torch.cat([x1, x2, x3, x4, x5], dim=1)
x = self.conv(x_cat)
- return x
-
-class HED(nn.Module):
+ return x
+
+
+class HED(torch.nn.Module):
"""
HED head module
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None):
super(HED, self).__init__()
- in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8, in_upsample_16 = in_channels_list
-
- self.conv1_2_16 = nn.Conv2d(in_conv_1_2_16,1,3,1,1)
+ (
+ in_conv_1_2_16,
+ in_upsample2,
+ in_upsample_4,
+ in_upsample_8,
+ in_upsample_16,
+ ) = in_channels_list
+
+ self.conv1_2_16 = torch.nn.Conv2d(in_conv_1_2_16, 1, 3, 1, 1)
# Upsample
- self.upsample2 = UpsampleCropBlock(in_upsample2,1,4,2,0)
- self.upsample4 = UpsampleCropBlock(in_upsample_4,1,8,4,0)
- self.upsample8 = UpsampleCropBlock(in_upsample_8,1,16,8,0)
- self.upsample16 = UpsampleCropBlock(in_upsample_16,1,32,16,0)
+ self.upsample2 = UpsampleCropBlock(in_upsample2, 1, 4, 2, 0)
+ self.upsample4 = UpsampleCropBlock(in_upsample_4, 1, 8, 4, 0)
+ self.upsample8 = UpsampleCropBlock(in_upsample_8, 1, 16, 8, 0)
+ self.upsample16 = UpsampleCropBlock(in_upsample_16, 1, 32, 16, 0)
# Concat and Fuse
self.concatfuse = ConcatFuseBlock()
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
-
+
Returns
-------
- :py:class:`torch.Tensor`
+ tensor : :py:class:`torch.Tensor`
"""
hw = x[0]
- conv1_2_16 = self.conv1_2_16(x[1])
- upsample2 = self.upsample2(x[2],hw)
- upsample4 = self.upsample4(x[3],hw)
- upsample8 = self.upsample8(x[4],hw)
- upsample16 = self.upsample16(x[5],hw)
- concatfuse = self.concatfuse(conv1_2_16,upsample2,upsample4,upsample8,upsample16)
-
- out = [upsample2,upsample4,upsample8,upsample16,concatfuse]
+ conv1_2_16 = self.conv1_2_16(x[1])
+ upsample2 = self.upsample2(x[2], hw)
+ upsample4 = self.upsample4(x[3], hw)
+ upsample8 = self.upsample8(x[4], hw)
+ upsample16 = self.upsample16(x[5], hw)
+ concatfuse = self.concatfuse(
+ conv1_2_16, upsample2, upsample4, upsample8, upsample16
+ )
+
+ out = [upsample2, upsample4, upsample8, upsample16, concatfuse]
return out
+
def build_hed():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+ module : :py:class:`torch.nn.Module`
"""
- backbone = vgg16(pretrained=False, return_features = [3, 8, 14, 22, 29])
+ backbone = vgg16(pretrained=False, return_features=[3, 8, 14, 22, 29])
hed_head = HED([64, 128, 256, 512, 512])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", hed_head)]))
- model.name = "HED"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", hed_head)])
+ )
+ model.name = "hed"
+ return model
diff --git a/bob/ip/binseg/modeling/losses.py b/bob/ip/binseg/modeling/losses.py
index de85a581515cb6bccb52074a68963e9978f2c72e..2f435c14cbaa3d33453a6c3b4c6d82a6d0588821 100644
--- a/bob/ip/binseg/modeling/losses.py
+++ b/bob/ip/binseg/modeling/losses.py
@@ -1,19 +1,40 @@
+"""Loss implementations"""
+
import torch
from torch.nn.modules.loss import _Loss
-from torch._jit_internal import weak_script_method
+# Conditionally decorates a method if a decorator exists in PyTorch
+# This overcomes an import error with versions of PyTorch >= 1.2, where the
+# decorator ``weak_script_method`` is not anymore available. See:
+# https://github.com/pytorch/pytorch/commit/10c4b98ade8349d841518d22f19a653a939e260c#diff-ee07db084d958260fd24b4b02d4f078d
+# from July 4th, 2019.
+try:
+ from torch._jit_internal import weak_script_method
+except ImportError:
+ def weak_script_method(x):
+ return x
class WeightedBCELogitsLoss(_Loss):
- """
- Implements Equation 1 in `Maninis et al. (2016)`_. Based on ``torch.nn.modules.loss.BCEWithLogitsLoss``.
+ """
+ Implements Equation 1 in [MANINIS-2016]_. Based on
+ :py:class:`torch.nn.BCEWithLogitsLoss`.
+
Calculate sum of weighted cross entropy loss.
"""
- def __init__(self, weight=None, size_average=None, reduce=None, reduction='mean', pos_weight=None):
+
+ def __init__(
+ self,
+ weight=None,
+ size_average=None,
+ reduce=None,
+ reduction="mean",
+ pos_weight=None,
+ ):
super(WeightedBCELogitsLoss, self).__init__(size_average, reduce, reduction)
- self.register_buffer('weight', weight)
- self.register_buffer('pos_weight', pos_weight)
+ self.register_buffer("weight", weight)
+ self.register_buffer("pos_weight", pos_weight)
@weak_script_method
def forward(self, input, target, masks=None):
@@ -23,37 +44,57 @@ class WeightedBCELogitsLoss(_Loss):
input : :py:class:`torch.Tensor`
target : :py:class:`torch.Tensor`
masks : :py:class:`torch.Tensor`, optional
-
+
Returns
-------
:py:class:`torch.Tensor`
"""
n, c, h, w = target.shape
- num_pos = torch.sum(target, dim=[1, 2, 3]).float().reshape(n,1) # torch.Size([n, 1])
- if hasattr(masks,'dtype'):
- num_mask_neg = c * h * w - torch.sum(masks, dim=[1, 2, 3]).float().reshape(n,1) # torch.Size([n, 1])
- num_neg = c * h * w - num_pos - num_mask_neg
+ num_pos = (
+ torch.sum(target, dim=[1, 2, 3]).float().reshape(n, 1)
+ ) # torch.Size([n, 1])
+ if hasattr(masks, "dtype"):
+ num_mask_neg = c * h * w - torch.sum(masks, dim=[1, 2, 3]).float().reshape(
+ n, 1
+ ) # torch.Size([n, 1])
+ num_neg = c * h * w - num_pos - num_mask_neg
else:
- num_neg = c * h * w - num_pos
- numposnumtotal = torch.ones_like(target) * (num_pos / (num_pos + num_neg)).unsqueeze(1).unsqueeze(2)
- numnegnumtotal = torch.ones_like(target) * (num_neg / (num_pos + num_neg)).unsqueeze(1).unsqueeze(2)
- weight = torch.where((target <= 0.5) , numposnumtotal, numnegnumtotal)
+ num_neg = c * h * w - num_pos
+ numposnumtotal = torch.ones_like(target) * (
+ num_pos / (num_pos + num_neg)
+ ).unsqueeze(1).unsqueeze(2)
+ numnegnumtotal = torch.ones_like(target) * (
+ num_neg / (num_pos + num_neg)
+ ).unsqueeze(1).unsqueeze(2)
+ weight = torch.where((target <= 0.5), numposnumtotal, numnegnumtotal)
+
+ loss = torch.nn.functional.binary_cross_entropy_with_logits(
+ input, target, weight=weight, reduction=self.reduction
+ )
+ return loss
- loss = torch.nn.functional.binary_cross_entropy_with_logits(input, target, weight=weight, reduction=self.reduction)
- return loss
class SoftJaccardBCELogitsLoss(_Loss):
- """
- Implements Equation 3 in `Iglovikov et al. (2018)`_. Based on ``torch.nn.modules.loss.BCEWithLogitsLoss``.
+ """
+ Implements Equation 3 in [IGLOVIKOV-2018]_. Based on
+ ``torch.nn.BCEWithLogitsLoss``.
Attributes
----------
alpha : float
determines the weighting of SoftJaccard and BCE. Default: ``0.7``
"""
- def __init__(self, alpha=0.7, size_average=None, reduce=None, reduction='mean', pos_weight=None):
- super(SoftJaccardBCELogitsLoss, self).__init__(size_average, reduce, reduction)
- self.alpha = alpha
+
+ def __init__(
+ self,
+ alpha=0.7,
+ size_average=None,
+ reduce=None,
+ reduction="mean",
+ pos_weight=None,
+ ):
+ super(SoftJaccardBCELogitsLoss, self).__init__(size_average, reduce, reduction)
+ self.alpha = alpha
@weak_script_method
def forward(self, input, target, masks=None):
@@ -63,7 +104,7 @@ class SoftJaccardBCELogitsLoss(_Loss):
input : :py:class:`torch.Tensor`
target : :py:class:`torch.Tensor`
masks : :py:class:`torch.Tensor`, optional
-
+
Returns
-------
:py:class:`torch.Tensor`
@@ -72,23 +113,35 @@ class SoftJaccardBCELogitsLoss(_Loss):
probabilities = torch.sigmoid(input)
intersection = (probabilities * target).sum()
sums = probabilities.sum() + target.sum()
-
- softjaccard = intersection/(sums - intersection + eps)
- bceloss = torch.nn.functional.binary_cross_entropy_with_logits(input, target, weight=None, reduction=self.reduction)
- loss = self.alpha * bceloss + (1 - self.alpha) * (1-softjaccard)
+ softjaccard = intersection / (sums - intersection + eps)
+
+ bceloss = torch.nn.functional.binary_cross_entropy_with_logits(
+ input, target, weight=None, reduction=self.reduction
+ )
+ loss = self.alpha * bceloss + (1 - self.alpha) * (1 - softjaccard)
return loss
class HEDWeightedBCELogitsLoss(_Loss):
- """
- Implements Equation 2 in `He et al. (2015)`_. Based on ``torch.nn.modules.loss.BCEWithLogitsLoss``.
+ """
+ Implements Equation 2 in [HE-2015]_. Based on
+ ``torch.nn.modules.loss.BCEWithLogitsLoss``.
+
Calculate sum of weighted cross entropy loss.
"""
- def __init__(self, weight=None, size_average=None, reduce=None, reduction='mean', pos_weight=None):
+
+ def __init__(
+ self,
+ weight=None,
+ size_average=None,
+ reduce=None,
+ reduction="mean",
+ pos_weight=None,
+ ):
super(HEDWeightedBCELogitsLoss, self).__init__(size_average, reduce, reduction)
- self.register_buffer('weight', weight)
- self.register_buffer('pos_weight', pos_weight)
+ self.register_buffer("weight", weight)
+ self.register_buffer("pos_weight", pos_weight)
@weak_script_method
def forward(self, inputlist, target, masks=None):
@@ -106,33 +159,57 @@ class HEDWeightedBCELogitsLoss(_Loss):
loss_over_all_inputs = []
for input in inputlist:
n, c, h, w = target.shape
- num_pos = torch.sum(target, dim=[1, 2, 3]).float().reshape(n,1) # torch.Size([n, 1])
- if hasattr(masks,'dtype'):
- num_mask_neg = c * h * w - torch.sum(masks, dim=[1, 2, 3]).float().reshape(n,1) # torch.Size([n, 1])
- num_neg = c * h * w - num_pos - num_mask_neg
- else:
+ num_pos = (
+ torch.sum(target, dim=[1, 2, 3]).float().reshape(n, 1)
+ ) # torch.Size([n, 1])
+ if hasattr(masks, "dtype"):
+ num_mask_neg = c * h * w - torch.sum(
+ masks, dim=[1, 2, 3]
+ ).float().reshape(
+ n, 1
+ ) # torch.Size([n, 1])
+ num_neg = c * h * w - num_pos - num_mask_neg
+ else:
num_neg = c * h * w - num_pos # torch.Size([n, 1])
- numposnumtotal = torch.ones_like(target) * (num_pos / (num_pos + num_neg)).unsqueeze(1).unsqueeze(2)
- numnegnumtotal = torch.ones_like(target) * (num_neg / (num_pos + num_neg)).unsqueeze(1).unsqueeze(2)
- weight = torch.where((target <= 0.5) , numposnumtotal, numnegnumtotal)
- loss = torch.nn.functional.binary_cross_entropy_with_logits(input, target, weight=weight, reduction=self.reduction)
+ numposnumtotal = torch.ones_like(target) * (
+ num_pos / (num_pos + num_neg)
+ ).unsqueeze(1).unsqueeze(2)
+ numnegnumtotal = torch.ones_like(target) * (
+ num_neg / (num_pos + num_neg)
+ ).unsqueeze(1).unsqueeze(2)
+ weight = torch.where((target <= 0.5), numposnumtotal, numnegnumtotal)
+ loss = torch.nn.functional.binary_cross_entropy_with_logits(
+ input, target, weight=weight, reduction=self.reduction
+ )
loss_over_all_inputs.append(loss.unsqueeze(0))
final_loss = torch.cat(loss_over_all_inputs).mean()
- return final_loss
+ return final_loss
class HEDSoftJaccardBCELogitsLoss(_Loss):
- """
- Implements Equation 3 in `Iglovikov et al. (2018)`_ for the hed network. Based on ``torch.nn.modules.loss.BCEWithLogitsLoss``.
+ """
+
+ Implements Equation 3 in [IGLOVIKOV-2018]_ for the hed network. Based on
+ :py:class:`torch.nn.BCEWithLogitsLoss`.
Attributes
----------
alpha : float
determines the weighting of SoftJaccard and BCE. Default: ``0.3``
"""
- def __init__(self, alpha=0.3, size_average=None, reduce=None, reduction='mean', pos_weight=None):
- super(HEDSoftJaccardBCELogitsLoss, self).__init__(size_average, reduce, reduction)
- self.alpha = alpha
+
+ def __init__(
+ self,
+ alpha=0.3,
+ size_average=None,
+ reduce=None,
+ reduction="mean",
+ pos_weight=None,
+ ):
+ super(HEDSoftJaccardBCELogitsLoss, self).__init__(
+ size_average, reduce, reduction
+ )
+ self.alpha = alpha
@weak_script_method
def forward(self, inputlist, target, masks=None):
@@ -142,7 +219,7 @@ class HEDSoftJaccardBCELogitsLoss(_Loss):
input : :py:class:`torch.Tensor`
target : :py:class:`torch.Tensor`
masks : :py:class:`torch.Tensor`, optional
-
+
Returns
-------
:py:class:`torch.Tensor`
@@ -153,48 +230,63 @@ class HEDSoftJaccardBCELogitsLoss(_Loss):
probabilities = torch.sigmoid(input)
intersection = (probabilities * target).sum()
sums = probabilities.sum() + target.sum()
-
- softjaccard = intersection/(sums - intersection + eps)
-
- bceloss = torch.nn.functional.binary_cross_entropy_with_logits(input, target, weight=None, reduction=self.reduction)
- loss = self.alpha * bceloss + (1 - self.alpha) * (1-softjaccard)
+
+ softjaccard = intersection / (sums - intersection + eps)
+
+ bceloss = torch.nn.functional.binary_cross_entropy_with_logits(
+ input, target, weight=None, reduction=self.reduction
+ )
+ loss = self.alpha * bceloss + (1 - self.alpha) * (1 - softjaccard)
loss_over_all_inputs.append(loss.unsqueeze(0))
final_loss = torch.cat(loss_over_all_inputs).mean()
- return loss
-
+ return final_loss
class MixJacLoss(_Loss):
- """
- Attributes
+ """
+
+ Parameters
----------
+
lambda_u : int
determines the weighting of SoftJaccard and BCE.
+
"""
- def __init__(self, lambda_u=100, jacalpha=0.7, size_average=None, reduce=None, reduction='mean', pos_weight=None):
+
+ def __init__(
+ self,
+ lambda_u=100,
+ jacalpha=0.7,
+ size_average=None,
+ reduce=None,
+ reduction="mean",
+ pos_weight=None,
+ ):
super(MixJacLoss, self).__init__(size_average, reduce, reduction)
self.lambda_u = lambda_u
self.labeled_loss = SoftJaccardBCELogitsLoss(alpha=jacalpha)
self.unlabeled_loss = torch.nn.BCEWithLogitsLoss()
-
@weak_script_method
def forward(self, input, target, unlabeled_input, unlabeled_traget, ramp_up_factor):
"""
Parameters
----------
+
input : :py:class:`torch.Tensor`
target : :py:class:`torch.Tensor`
unlabeled_input : :py:class:`torch.Tensor`
unlabeled_traget : :py:class:`torch.Tensor`
ramp_up_factor : float
-
+
Returns
-------
+
list
+
"""
- ll = self.labeled_loss(input,target)
+ ll = self.labeled_loss(input, target)
ul = self.unlabeled_loss(unlabeled_input, unlabeled_traget)
-
+
loss = ll + self.lambda_u * ramp_up_factor * ul
- return loss, ll, ul
\ No newline at end of file
+ return loss, ll, ul
diff --git a/bob/ip/binseg/modeling/m2u.py b/bob/ip/binseg/modeling/m2u.py
index 7db86168c0b6f703546de4dca2e22539e73adeb4..8861b965e3294a472da5edf54d23d215536f8c0d 100644
--- a/bob/ip/binseg/modeling/m2u.py
+++ b/bob/ip/binseg/modeling/m2u.py
@@ -5,99 +5,116 @@
from collections import OrderedDict
import torch
-from torch import nn
+import torch.nn
from bob.ip.binseg.modeling.backbones.mobilenetv2 import MobileNetV2, InvertedResidual
-class DecoderBlock(nn.Module):
+
+class DecoderBlock(torch.nn.Module):
"""
Decoder block: upsample and concatenate with features maps from the encoder part
"""
- def __init__(self,up_in_c,x_in_c,upsamplemode='bilinear',expand_ratio=0.15):
+
+ def __init__(self, up_in_c, x_in_c, upsamplemode="bilinear", expand_ratio=0.15):
super().__init__()
- self.upsample = nn.Upsample(scale_factor=2,mode=upsamplemode,align_corners=False) # H, W -> 2H, 2W
- self.ir1 = InvertedResidual(up_in_c+x_in_c,(x_in_c + up_in_c) // 2,stride=1,expand_ratio=expand_ratio)
+ self.upsample = torch.nn.Upsample(
+ scale_factor=2, mode=upsamplemode, align_corners=False
+ ) # H, W -> 2H, 2W
+ self.ir1 = InvertedResidual(
+ up_in_c + x_in_c,
+ (x_in_c + up_in_c) // 2,
+ stride=1,
+ expand_ratio=expand_ratio,
+ )
- def forward(self,up_in,x_in):
+ def forward(self, up_in, x_in):
up_out = self.upsample(up_in)
- cat_x = torch.cat([up_out, x_in] , dim=1)
+ cat_x = torch.cat([up_out, x_in], dim=1)
x = self.ir1(cat_x)
return x
-
-class LastDecoderBlock(nn.Module):
- def __init__(self,x_in_c,upsamplemode='bilinear',expand_ratio=0.15):
+
+
+class LastDecoderBlock(torch.nn.Module):
+ def __init__(self, x_in_c, upsamplemode="bilinear", expand_ratio=0.15):
super().__init__()
- self.upsample = nn.Upsample(scale_factor=2,mode=upsamplemode,align_corners=False) # H, W -> 2H, 2W
- self.ir1 = InvertedResidual(x_in_c,1,stride=1,expand_ratio=expand_ratio)
+ self.upsample = torch.nn.Upsample(
+ scale_factor=2, mode=upsamplemode, align_corners=False
+ ) # H, W -> 2H, 2W
+ self.ir1 = InvertedResidual(x_in_c, 1, stride=1, expand_ratio=expand_ratio)
- def forward(self,up_in,x_in):
+ def forward(self, up_in, x_in):
up_out = self.upsample(up_in)
- cat_x = torch.cat([up_out, x_in] , dim=1)
+ cat_x = torch.cat([up_out, x_in], dim=1)
x = self.ir1(cat_x)
return x
-
-class M2U(nn.Module):
+class M2U(torch.nn.Module):
"""
M2U-Net head module
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
- def __init__(self, in_channels_list=None,upsamplemode='bilinear',expand_ratio=0.15):
+
+ def __init__(
+ self, in_channels_list=None, upsamplemode="bilinear", expand_ratio=0.15
+ ):
super(M2U, self).__init__()
# Decoder
- self.decode4 = DecoderBlock(96,32,upsamplemode,expand_ratio)
- self.decode3 = DecoderBlock(64,24,upsamplemode,expand_ratio)
- self.decode2 = DecoderBlock(44,16,upsamplemode,expand_ratio)
- self.decode1 = LastDecoderBlock(33,upsamplemode,expand_ratio)
-
- # initilaize weights
+ self.decode4 = DecoderBlock(96, 32, upsamplemode, expand_ratio)
+ self.decode3 = DecoderBlock(64, 24, upsamplemode, expand_ratio)
+ self.decode2 = DecoderBlock(44, 16, upsamplemode, expand_ratio)
+ self.decode1 = LastDecoderBlock(33, upsamplemode, expand_ratio)
+
+ # initilaize weights
self._initialize_weights()
def _initialize_weights(self):
for m in self.modules():
- if isinstance(m, nn.Conv2d):
- nn.init.kaiming_uniform_(m.weight, a=1)
+ if isinstance(m, torch.nn.Conv2d):
+ torch.nn.init.kaiming_uniform_(m.weight, a=1)
if m.bias is not None:
- nn.init.constant_(m.bias, 0)
- elif isinstance(m, nn.BatchNorm2d):
+ torch.nn.init.constant_(m.bias, 0)
+ elif isinstance(m, torch.nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
-
- def forward(self,x):
+
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
Returns
-------
- :py:class:`torch.Tensor`
+ tensor : :py:class:`torch.Tensor`
"""
- decode4 = self.decode4(x[5],x[4]) # 96, 32
- decode3 = self.decode3(decode4,x[3]) # 64, 24
- decode2 = self.decode2(decode3,x[2]) # 44, 16
- decode1 = self.decode1(decode2,x[1]) # 30, 3
-
+ decode4 = self.decode4(x[5], x[4]) # 96, 32
+ decode3 = self.decode3(decode4, x[3]) # 64, 24
+ decode2 = self.decode2(decode3, x[2]) # 44, 16
+ decode1 = self.decode1(decode2, x[1]) # 30, 3
+
return decode1
+
def build_m2unet():
- """
+ """
Adds backbone and head together
Returns
-------
- :py:class:torch.nn.Module
+ module : :py:class:`torch.nn.Module`
"""
- backbone = MobileNetV2(return_features = [1, 3, 6, 13], m2u=True)
+ backbone = MobileNetV2(return_features=[1, 3, 6, 13], m2u=True)
m2u_head = M2U(in_channels_list=[16, 24, 32, 96])
- model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", m2u_head)]))
- model.name = "M2UNet"
- return model
\ No newline at end of file
+ model = torch.nn.Sequential(
+ OrderedDict([("backbone", backbone), ("head", m2u_head)])
+ )
+ model.name = "m2unet"
+ return model
diff --git a/bob/ip/binseg/modeling/make_layers.py b/bob/ip/binseg/modeling/make_layers.py
index 7e3984433273eaa0d7f86b3e720682c9460552f3..23704eae10913ad9235a19cde4f024587333ddac 100644
--- a/bob/ip/binseg/modeling/make_layers.py
+++ b/bob/ip/binseg/modeling/make_layers.py
@@ -2,76 +2,108 @@
# -*- coding: utf-8 -*-
import torch
-import torch.nn as nn
+import torch.nn
from torch.nn import Conv2d
from torch.nn import ConvTranspose2d
-def conv_with_kaiming_uniform(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1):
+
+def conv_with_kaiming_uniform(
+ in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1
+):
conv = Conv2d(
- in_channels,
- out_channels,
- kernel_size=kernel_size,
- stride=stride,
- padding=padding,
- dilation=dilation,
- bias= True
- )
- # Caffe2 implementation uses XavierFill, which in fact
- # corresponds to kaiming_uniform_ in PyTorch
- nn.init.kaiming_uniform_(conv.weight, a=1)
- nn.init.constant_(conv.bias, 0)
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=True,
+ )
+ # Caffe2 implementation uses XavierFill, which in fact
+ # corresponds to kaiming_uniform_ in PyTorch
+ torch.nn.init.kaiming_uniform_(conv.weight, a=1)
+ torch.nn.init.constant_(conv.bias, 0)
return conv
-def convtrans_with_kaiming_uniform(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1):
+def convtrans_with_kaiming_uniform(
+ in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1
+):
conv = ConvTranspose2d(
- in_channels,
- out_channels,
- kernel_size=kernel_size,
- stride=stride,
- padding=padding,
- dilation=dilation,
- bias= True
- )
- # Caffe2 implementation uses XavierFill, which in fact
- # corresponds to kaiming_uniform_ in PyTorch
- nn.init.kaiming_uniform_(conv.weight, a=1)
- nn.init.constant_(conv.bias, 0)
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=True,
+ )
+ # Caffe2 implementation uses XavierFill, which in fact
+ # corresponds to kaiming_uniform_ in PyTorch
+ torch.nn.init.kaiming_uniform_(conv.weight, a=1)
+ torch.nn.init.constant_(conv.bias, 0)
return conv
-class UpsampleCropBlock(nn.Module):
- def __init__(self, in_channels, out_channels, up_kernel_size, up_stride, up_padding, pixelshuffle=False):
- """
- Combines Conv2d, ConvTransposed2d and Cropping. Simulates the caffe2 crop layer in the forward function.
- Used for DRIU and HED.
-
- Attributes
- ----------
- in_channels : number of channels of intermediate layer
- out_channels : number of output channels
- up_kernel_size : kernel size for transposed convolution
- up_stride : stride for transposed convolution
- up_padding : padding for transposed convolution
- """
+class UpsampleCropBlock(torch.nn.Module):
+ """
+ Combines Conv2d, ConvTransposed2d and Cropping. Simulates the caffe2 crop
+ layer in the forward function.
+
+ Used for DRIU and HED.
+
+ Parameters
+ ----------
+
+ in_channels : int
+ number of channels of intermediate layer
+ out_channels : int
+ number of output channels
+ up_kernel_size : int
+ kernel size for transposed convolution
+ up_stride : int
+ stride for transposed convolution
+ up_padding : int
+ padding for transposed convolution
+
+ """
+
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ up_kernel_size,
+ up_stride,
+ up_padding,
+ pixelshuffle=False,
+ ):
super().__init__()
- # NOTE: Kaiming init, replace with nn.Conv2d and nn.ConvTranspose2d to get original DRIU impl.
+ # NOTE: Kaiming init, replace with torch.nn.Conv2d and torch.nn.ConvTranspose2d to get original DRIU impl.
self.conv = conv_with_kaiming_uniform(in_channels, out_channels, 3, 1, 1)
if pixelshuffle:
- self.upconv = PixelShuffle_ICNR( out_channels, out_channels, scale = up_stride)
+ self.upconv = PixelShuffle_ICNR(out_channels, out_channels, scale=up_stride)
else:
- self.upconv = convtrans_with_kaiming_uniform(out_channels, out_channels, up_kernel_size, up_stride, up_padding)
-
-
+ self.upconv = convtrans_with_kaiming_uniform(
+ out_channels, out_channels, up_kernel_size, up_stride, up_padding
+ )
+
def forward(self, x, input_res):
- """
- Forward pass of UpsampleBlock. Upsampled feature maps are cropped to the resolution of the input image.
- Attributes
+ """Forward pass of UpsampleBlock.
+
+ Upsampled feature maps are cropped to the resolution of the input
+ image.
+
+ Parameters
----------
- x : input channels
- input_res : tuple (h,w)
- Resolution of the input image
+
+ x : tuple
+ input channels
+
+ input_res : tuple
+ Resolution of the input image format ``(height, width)``
+
"""
+
img_h = input_res[0]
img_w = input_res[1]
x = self.conv(x)
@@ -80,84 +112,93 @@ class UpsampleCropBlock(nn.Module):
# height
up_h = x.shape[2]
h_crop = up_h - img_h
- h_s = h_crop//2
+ h_s = h_crop // 2
h_e = up_h - (h_crop - h_s)
# width
up_w = x.shape[3]
- w_crop = up_w-img_w
- w_s = w_crop//2
+ w_crop = up_w - img_w
+ w_s = w_crop // 2
w_e = up_w - (w_crop - w_s)
- # perform crop
- # needs explicit ranges for onnx export
- x = x[:,:,h_s:h_e,w_s:w_e] # crop to input size
-
- return x
+ # perform crop
+ # needs explicit ranges for onnx export
+ x = x[:, :, h_s:h_e, w_s:w_e] # crop to input size
+ return x
def ifnone(a, b):
- "`a` if `a` is not None, otherwise `b`."
+ "``a`` if ``a`` is not None, otherwise ``b``."
return b if a is None else a
-def icnr(x, scale=2, init=nn.init.kaiming_normal_):
- """
- https://docs.fast.ai/layers.html#PixelShuffle_ICNR
- ICNR init of `x`, with `scale` and `init` function.
+
+def icnr(x, scale=2, init=torch.nn.init.kaiming_normal_):
+ """https://docs.fast.ai/layers.html#PixelShuffle_ICNR
+
+ ICNR init of ``x``, with ``scale`` and ``init`` function.
"""
- ni,nf,h,w = x.shape
- ni2 = int(ni/(scale**2))
- k = init(torch.zeros([ni2,nf,h,w])).transpose(0, 1)
+
+ ni, nf, h, w = x.shape
+ ni2 = int(ni / (scale ** 2))
+ k = init(torch.zeros([ni2, nf, h, w])).transpose(0, 1)
k = k.contiguous().view(ni2, nf, -1)
- k = k.repeat(1, 1, scale**2)
- k = k.contiguous().view([nf,ni,h,w]).transpose(0, 1)
+ k = k.repeat(1, 1, scale ** 2)
+ k = k.contiguous().view([nf, ni, h, w]).transpose(0, 1)
x.data.copy_(k)
-class PixelShuffle_ICNR(nn.Module):
- """
- https://docs.fast.ai/layers.html#PixelShuffle_ICNR
- Upsample by `scale` from `ni` filters to `nf` (default `ni`), using `nn.PixelShuffle`, `icnr` init, and `weight_norm`.
+
+class PixelShuffle_ICNR(torch.nn.Module):
+ """https://docs.fast.ai/layers.html#PixelShuffle_ICNR
+
+ Upsample by ``scale`` from ``ni`` filters to ``nf`` (default ``ni``), using
+ ``torch.nn.PixelShuffle``, ``icnr`` init, and ``weight_norm``.
"""
- def __init__(self, ni:int, nf:int=None, scale:int=2):
+
+ def __init__(self, ni: int, nf: int = None, scale: int = 2):
super().__init__()
nf = ifnone(nf, ni)
- self.conv = conv_with_kaiming_uniform(ni, nf*(scale**2), 1)
+ self.conv = conv_with_kaiming_uniform(ni, nf * (scale ** 2), 1)
icnr(self.conv.weight)
- self.shuf = nn.PixelShuffle(scale)
+ self.shuf = torch.nn.PixelShuffle(scale)
# Blurring over (h*w) kernel
# "Super-Resolution using Convolutional Neural Networks without Any Checkerboard Artifacts"
# - https://arxiv.org/abs/1806.02658
- self.pad = nn.ReplicationPad2d((1,0,1,0))
- self.blur = nn.AvgPool2d(2, stride=1)
- self.relu = nn.ReLU(inplace=True)
+ self.pad = torch.nn.ReplicationPad2d((1, 0, 1, 0))
+ self.blur = torch.nn.AvgPool2d(2, stride=1)
+ self.relu = torch.nn.ReLU(inplace=True)
- def forward(self,x):
+ def forward(self, x):
x = self.shuf(self.relu(self.conv(x)))
x = self.blur(self.pad(x))
return x
-class UnetBlock(nn.Module):
+
+class UnetBlock(torch.nn.Module):
def __init__(self, up_in_c, x_in_c, pixel_shuffle=False, middle_block=False):
super().__init__()
# middle block for VGG based U-Net
if middle_block:
- up_out_c = up_in_c
+ up_out_c = up_in_c
else:
- up_out_c = up_in_c // 2
+ up_out_c = up_in_c // 2
cat_channels = x_in_c + up_out_c
inner_channels = cat_channels // 2
-
+
if pixel_shuffle:
- self.upsample = PixelShuffle_ICNR( up_in_c, up_out_c )
+ self.upsample = PixelShuffle_ICNR(up_in_c, up_out_c)
else:
- self.upsample = convtrans_with_kaiming_uniform( up_in_c, up_out_c, 2, 2)
- self.convtrans1 = convtrans_with_kaiming_uniform( cat_channels, inner_channels, 3, 1, 1)
- self.convtrans2 = convtrans_with_kaiming_uniform( inner_channels, inner_channels, 3, 1, 1)
- self.relu = nn.ReLU(inplace=True)
+ self.upsample = convtrans_with_kaiming_uniform(up_in_c, up_out_c, 2, 2)
+ self.convtrans1 = convtrans_with_kaiming_uniform(
+ cat_channels, inner_channels, 3, 1, 1
+ )
+ self.convtrans2 = convtrans_with_kaiming_uniform(
+ inner_channels, inner_channels, 3, 1, 1
+ )
+ self.relu = torch.nn.ReLU(inplace=True)
def forward(self, up_in, x_in):
up_out = self.upsample(up_in)
- cat_x = torch.cat([up_out, x_in] , dim=1)
+ cat_x = torch.cat([up_out, x_in], dim=1)
x = self.relu(self.convtrans1(cat_x))
x = self.relu(self.convtrans2(x))
- return x
\ No newline at end of file
+ return x
diff --git a/bob/ip/binseg/modeling/resunet.py b/bob/ip/binseg/modeling/resunet.py
index 38f66cddf787b8e48f8d7f8aeb50001121c229eb..cce8242ec957dc9814186ee5967b5cb3ae57b5cd 100644
--- a/bob/ip/binseg/modeling/resunet.py
+++ b/bob/ip/binseg/modeling/resunet.py
@@ -2,28 +2,32 @@
# -*- coding: utf-8 -*-
import torch.nn as nn
-import torch
from collections import OrderedDict
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform, convtrans_with_kaiming_uniform, PixelShuffle_ICNR, UnetBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ PixelShuffle_ICNR,
+ UnetBlock,
+)
from bob.ip.binseg.modeling.backbones.resnet import resnet50
-
class ResUNet(nn.Module):
"""
UNet head module for ResNet backbones
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None, pixel_shuffle=False):
super(ResUNet, self).__init__()
# number of channels
c_decode1, c_decode2, c_decode3, c_decode4, c_decode5 = in_channels_list
# number of channels for last upsampling operation
- c_decode0 = (c_decode1 + c_decode2//2)//2
+ c_decode0 = (c_decode1 + c_decode2 // 2) // 2
# build layers
self.decode4 = UnetBlock(c_decode5, c_decode4, pixel_shuffle)
@@ -36,34 +40,35 @@ class ResUNet(nn.Module):
self.decode0 = convtrans_with_kaiming_uniform(c_decode0, c_decode0, 2, 2)
self.final = conv_with_kaiming_uniform(c_decode0, 1, 1)
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
"""
# NOTE: x[0]: height and width of input image not needed in U-Net architecture
- decode4 = self.decode4(x[5], x[4])
- decode3 = self.decode3(decode4, x[3])
- decode2 = self.decode2(decode3, x[2])
- decode1 = self.decode1(decode2, x[1])
+ decode4 = self.decode4(x[5], x[4])
+ decode3 = self.decode3(decode4, x[3])
+ decode2 = self.decode2(decode3, x[2])
+ decode1 = self.decode1(decode2, x[1])
decode0 = self.decode0(decode1)
out = self.final(decode0)
return out
+
def build_res50unet():
- """
+ """
Adds backbone and head together
Returns
-------
- model : :py:class:torch.nn.Module
+ model : :py:class:`torch.nn.Module`
"""
- backbone = resnet50(pretrained=False, return_features = [2, 4, 5, 6, 7])
- unet_head = ResUNet([64, 256, 512, 1024, 2048],pixel_shuffle=False)
+ backbone = resnet50(pretrained=False, return_features=[2, 4, 5, 6, 7])
+ unet_head = ResUNet([64, 256, 512, 1024, 2048], pixel_shuffle=False)
model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", unet_head)]))
- model.name = "ResUNet"
- return model
\ No newline at end of file
+ model.name = "resunet"
+ return model
diff --git a/bob/ip/binseg/modeling/unet.py b/bob/ip/binseg/modeling/unet.py
index d1102592b74d2ea2c8af8ea2657ac6f1775a92d7..ac3f1d5850a066da01b20cf37620c29c2cb20e62 100644
--- a/bob/ip/binseg/modeling/unet.py
+++ b/bob/ip/binseg/modeling/unet.py
@@ -2,27 +2,31 @@
# -*- coding: utf-8 -*-
import torch.nn as nn
-import torch
from collections import OrderedDict
-from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform, convtrans_with_kaiming_uniform, PixelShuffle_ICNR, UnetBlock
+from bob.ip.binseg.modeling.make_layers import (
+ conv_with_kaiming_uniform,
+ convtrans_with_kaiming_uniform,
+ PixelShuffle_ICNR,
+ UnetBlock,
+)
from bob.ip.binseg.modeling.backbones.vgg import vgg16
-
class UNet(nn.Module):
"""
UNet head module
-
+
Parameters
----------
in_channels_list : list
number of channels for each feature map that is returned from backbone
"""
+
def __init__(self, in_channels_list=None, pixel_shuffle=False):
super(UNet, self).__init__()
# number of channels
c_decode1, c_decode2, c_decode3, c_decode4, c_decode5 = in_channels_list
-
+
# build layers
self.decode4 = UnetBlock(c_decode5, c_decode4, pixel_shuffle, middle_block=True)
self.decode3 = UnetBlock(c_decode4, c_decode3, pixel_shuffle)
@@ -30,34 +34,36 @@ class UNet(nn.Module):
self.decode1 = UnetBlock(c_decode2, c_decode1, pixel_shuffle)
self.final = conv_with_kaiming_uniform(c_decode1, 1, 1)
- def forward(self,x):
+ def forward(self, x):
"""
Parameters
----------
x : list
list of tensors as returned from the backbone network.
- First element: height and width of input image.
+ First element: height and width of input image.
Remaining elements: feature maps for each feature level.
"""
# NOTE: x[0]: height and width of input image not needed in U-Net architecture
- decode4 = self.decode4(x[5], x[4])
- decode3 = self.decode3(decode4, x[3])
- decode2 = self.decode2(decode3, x[2])
- decode1 = self.decode1(decode2, x[1])
+ decode4 = self.decode4(x[5], x[4])
+ decode3 = self.decode3(decode4, x[3])
+ decode2 = self.decode2(decode3, x[2])
+ decode1 = self.decode1(decode2, x[1])
out = self.final(decode1)
return out
+
def build_unet():
- """
+ """
Adds backbone and head together
Returns
-------
- model : :py:class:torch.nn.Module
+ module : :py:class:`torch.nn.Module`
"""
- backbone = vgg16(pretrained=False, return_features = [3, 8, 14, 22, 29])
+
+ backbone = vgg16(pretrained=False, return_features=[3, 8, 14, 22, 29])
unet_head = UNet([64, 128, 256, 512, 512], pixel_shuffle=False)
model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", unet_head)]))
model.name = "UNet"
- return model
\ No newline at end of file
+ return model
diff --git a/bob/ip/binseg/script/__init__.py b/bob/ip/binseg/script/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/script/__init__.py
+++ b/bob/ip/binseg/script/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/script/analyze.py b/bob/ip/binseg/script/analyze.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd66611d635c5a31b7163c0b69eb9da1ee5e955e
--- /dev/null
+++ b/bob/ip/binseg/script/analyze.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+
+import click
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ ConfigCommand,
+ ResourceOption,
+)
+
+from .binseg import save_sh_command
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+@click.command(
+ entry_point_group="bob.ip.binseg.config",
+ cls=ConfigCommand,
+ epilog="""Examples:
+
+\b
+ 1. Re-evaluates a pre-trained M2U-Net model with DRIVE (vessel
+ segmentation), on the CPU, by running inference and evaluation on results
+ from its test set:
+
+ $ bob binseg analyze -vv m2unet drive --weight=model.path
+
+""",
+)
+@click.option(
+ "--output-folder",
+ "-o",
+ help="Path where to store experiment outputs (created if does not exist)",
+ required=True,
+ type=click.Path(),
+ default="results",
+ cls=ResourceOption,
+)
+@click.option(
+ "--model",
+ "-m",
+ help="A torch.nn.Module instance implementing the network to be trained, and then evaluated",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--dataset",
+ "-d",
+ help="A dictionary mapping string keys to "
+ "bob.ip.binseg.data.utils.SampleList2TorchDataset's. At least one key "
+ "named 'train' must be available. This dataset will be used for training "
+ "the network model. All other datasets will be used for prediction and "
+ "evaluation. Dataset descriptions include all required pre-processing, "
+ "including eventual data augmentation, which may be eventually excluded "
+ "for prediction and evaluation purposes",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--second-annotator",
+ "-S",
+ help="A dataset or dictionary, like in --dataset, with the same "
+ "sample keys, but with annotations from a different annotator that is "
+ "going to be compared to the one in --dataset",
+ required=False,
+ default=None,
+ cls=ResourceOption,
+ show_default=True,
+)
+@click.option(
+ "--batch-size",
+ "-b",
+ help="Number of samples in every batch (this parameter affects "
+ "memory requirements for the network). If the number of samples in "
+ "the batch is larger than the total number of samples available for "
+ "training, this value is truncated. If this number is smaller, then "
+ "batches of the specified size are created and fed to the network "
+ "until there are no more new samples to feed (epoch is finished). "
+ "If the total number of training samples is not a multiple of the "
+ "batch-size, the last batch will be smaller than the first.",
+ required=True,
+ show_default=True,
+ default=1,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--device",
+ "-d",
+ help='A string indicating the device to use (e.g. "cpu" or "cuda:0")',
+ show_default=True,
+ required=True,
+ default="cpu",
+ cls=ResourceOption,
+)
+@click.option(
+ "--overlayed/--no-overlayed",
+ "-O",
+ help="Creates overlayed representations of the output probability maps, "
+ "similar to --overlayed in prediction-mode, except it includes "
+ "distinctive colours for true and false positives and false negatives. "
+ "If not set, or empty then do **NOT** output overlayed images.",
+ show_default=True,
+ default=False,
+ required=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--weight",
+ "-w",
+ help="Path or URL to pretrained model file (.pth extension)",
+ required=True,
+ cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+@click.pass_context
+def analyze(
+ ctx,
+ model,
+ output_folder,
+ batch_size,
+ dataset,
+ second_annotator,
+ device,
+ overlayed,
+ weight,
+ verbose,
+ **kwargs,
+):
+ """Runs a complete evaluation from prediction to comparison
+
+ This script is just a wrapper around the individual scripts for running
+ prediction and evaluating FCN models. It organises the output in a
+ preset way::
+
+\b
+ └─ <output-folder>/
+ ├── predictions/ #the prediction outputs for the train/test set
+ ├── overlayed/ #the overlayed outputs for the train/test set
+ ├── predictions/ #predictions overlayed on the input images
+ ├── analysis/ #predictions overlayed on the input images
+ ├ #including analysis of false positives, negatives
+ ├ #and true positives
+ └── second-annotator/ #if set, store overlayed images for the
+ #second annotator here
+ └── analysis / #the outputs of the analysis of both train/test sets
+ #includes second-annotator "metrics" as well, if
+ # configured
+
+ N.B.: The tool is designed to prevent analysis bias and allows one to
+ provide separate subsets for training and evaluation. Instead of using
+ simple datasets, datasets for full experiment running should be
+ dictionaries with specific subset names:
+
+ * ``__train__``: dataset used for training, prioritarily. It is typically
+ the dataset containing data augmentation pipelines.
+ * ``train`` (optional): a copy of the ``__train__`` dataset, without data
+ augmentation, that will be evaluated alongside other sets available
+ * ``*``: any other name, not starting with an underscore character (``_``),
+ will be considered a test set for evaluation.
+
+ N.B.2: The threshold used for calculating the F1-score on the test set, or
+ overlay analysis (false positives, negatives and true positives overprinted
+ on the original image) also follows the logic above.
+ """
+
+ command_sh = os.path.join(output_folder, "command.sh")
+ if not os.path.exists(command_sh):
+ # only save if experiment has not saved yet something similar
+ save_sh_command(command_sh)
+
+
+
+ ## Prediction
+ logger.info("Started prediction")
+
+ from .predict import predict
+
+ predictions_folder = os.path.join(output_folder, "predictions")
+ overlayed_folder = (
+ os.path.join(output_folder, "overlayed", "predictions")
+ if overlayed
+ else None
+ )
+
+ ctx.invoke(
+ predict,
+ output_folder=predictions_folder,
+ model=model,
+ dataset=dataset,
+ batch_size=batch_size,
+ device=device,
+ weight=weight,
+ overlayed=overlayed_folder,
+ verbose=verbose,
+ )
+ logger.info("Ended prediction")
+
+ ## Evaluation
+ logger.info("Started evaluation")
+
+ from .evaluate import evaluate
+
+ overlayed_folder = (
+ os.path.join(output_folder, "overlayed", "analysis")
+ if overlayed
+ else None
+ )
+
+ # choosing the overlayed_threshold
+ if "validation" in dataset:
+ threshold = "validation"
+ elif "train" in dataset:
+ threshold = "train"
+ else:
+ threshold = 0.5
+ logger.info(f"Setting --threshold={threshold}...")
+
+ analysis_folder = os.path.join(output_folder, "analysis")
+ ctx.invoke(
+ evaluate,
+ output_folder=analysis_folder,
+ predictions_folder=predictions_folder,
+ dataset=dataset,
+ second_annotator=second_annotator,
+ overlayed=overlayed_folder,
+ threshold=threshold,
+ verbose=verbose,
+ )
+
+ logger.info("Ended evaluation")
+
+ ## Comparison
+ logger.info("Started comparison")
+
+ # compare performances on the various sets
+ from .compare import compare
+
+ systems = []
+ for k, v in dataset.items():
+ if k.startswith("_"):
+ logger.info(f"Skipping dataset '{k}' (not to be compared)")
+ continue
+ systems += [k, os.path.join(analysis_folder, f"{k}.csv")]
+ if second_annotator is not None:
+ for k, v in second_annotator.items():
+ if k.startswith("_"):
+ logger.info(f"Skipping dataset '{k}' (not to be compared)")
+ continue
+ systems += [
+ f"{k} (2nd. annot.)",
+ os.path.join(
+ analysis_folder, "second-annotator", f"{k}.csv"
+ ),
+ ]
+
+ output_figure = os.path.join(output_folder, "comparison.pdf")
+ output_table = os.path.join(output_folder, "comparison.rst")
+
+ ctx.invoke(
+ compare,
+ label_path=systems,
+ output_figure=output_figure,
+ output_table=output_table,
+ threshold=threshold,
+ verbose=verbose,
+ )
+
+ logger.info("Ended comparison")
diff --git a/bob/ip/binseg/script/binseg.py b/bob/ip/binseg/script/binseg.py
index 944a99953a40ed2fc4c3a0a9d07e28e8f0f21c5e..5fea88b1ffc244b4a6a55d022cbb666332de7e2b 100644
--- a/bob/ip/binseg/script/binseg.py
+++ b/bob/ip/binseg/script/binseg.py
@@ -3,628 +3,121 @@
"""The main entry for bob ip binseg (click-based) scripts."""
-
import os
+import sys
import time
-import numpy
-import collections
-import pkg_resources
-import glob
+import tempfile
+import urllib.request
+import pkg_resources
import click
from click_plugins import with_plugins
+from tqdm import tqdm
-import logging
-import torch
-
-import bob.extension
-from bob.extension.scripts.click_helper import (verbosity_option,
- ConfigCommand, ResourceOption, AliasedGroup)
-
-from bob.ip.binseg.utils.checkpointer import DetectronCheckpointer
-from torch.utils.data import DataLoader
-from bob.ip.binseg.engine.trainer import do_train
-from bob.ip.binseg.engine.ssltrainer import do_ssltrain
-from bob.ip.binseg.engine.inferencer import do_inference
-from bob.ip.binseg.utils.plot import plot_overview
-from bob.ip.binseg.utils.click import OptionEatAll
-from bob.ip.binseg.utils.rsttable import create_overview_grid
-from bob.ip.binseg.utils.plot import metricsviz, overlay,savetransformedtest
-from bob.ip.binseg.utils.transformfolder import transformfolder as transfld
-from bob.ip.binseg.utils.evaluate import do_eval
-from bob.ip.binseg.engine.predicter import do_predict
+from bob.extension.scripts.click_helper import AliasedGroup
+import logging
logger = logging.getLogger(__name__)
-@with_plugins(pkg_resources.iter_entry_points('bob.ip.binseg.cli'))
-@click.group(cls=AliasedGroup)
-def binseg():
- """Binary 2D Fundus Image Segmentation Benchmark commands."""
- pass
-
-# Train
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- default="output",
- cls=ResourceOption
- )
-@click.option(
- '--model',
- '-m',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--optimizer',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--criterion',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--scheduler',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--pretrained-backbone',
- '-t',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--batch-size',
- '-b',
- required=True,
- default=2,
- cls=ResourceOption)
-@click.option(
- '--epochs',
- '-e',
- help='Number of epochs used for training',
- show_default=True,
- required=True,
- default=1000,
- cls=ResourceOption)
-@click.option(
- '--checkpoint-period',
- '-p',
- help='Number of epochs after which a checkpoint is saved',
- show_default=True,
- required=True,
- default=100,
- cls=ResourceOption)
-@click.option(
- '--device',
- '-d',
- help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
- show_default=True,
- required=True,
- default='cpu',
- cls=ResourceOption)
-@click.option(
- '--seed',
- '-s',
- help='torch random seed',
- show_default=True,
- required=False,
- default=42,
- cls=ResourceOption)
-
-@verbosity_option(cls=ResourceOption)
-def train(model
- ,optimizer
- ,scheduler
- ,output_path
- ,epochs
- ,pretrained_backbone
- ,batch_size
- ,criterion
- ,dataset
- ,checkpoint_period
- ,device
- ,seed
- ,**kwargs):
- """ Train a model """
-
- if not os.path.exists(output_path): os.makedirs(output_path)
- torch.manual_seed(seed)
- # PyTorch dataloader
- data_loader = DataLoader(
- dataset = dataset
- ,batch_size = batch_size
- ,shuffle= True
- ,pin_memory = torch.cuda.is_available()
- )
-
- # Checkpointer
- checkpointer = DetectronCheckpointer(model, optimizer, scheduler,save_dir = output_path, save_to_disk=True)
- arguments = {}
- arguments["epoch"] = 0
- extra_checkpoint_data = checkpointer.load(pretrained_backbone)
- arguments.update(extra_checkpoint_data)
- arguments["max_epoch"] = epochs
-
- # Train
- logger.info("Training for {} epochs".format(arguments["max_epoch"]))
- logger.info("Continuing from epoch {}".format(arguments["epoch"]))
- do_train(model
- , data_loader
- , optimizer
- , criterion
- , scheduler
- , checkpointer
- , checkpoint_period
- , device
- , arguments
- , output_path
- )
-
-
-# Inference
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- default="output",
- cls=ResourceOption
- )
-@click.option(
- '--model',
- '-m',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--batch-size',
- '-b',
- required=True,
- default=2,
- cls=ResourceOption)
-@click.option(
- '--device',
- '-d',
- help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
- show_default=True,
- required=True,
- default='cpu',
- cls=ResourceOption)
-@click.option(
- '--weight',
- '-w',
- help='Path or URL to pretrained model',
- required=False,
- default=None,
- cls=ResourceOption
- )
-@verbosity_option(cls=ResourceOption)
-def test(model
- ,output_path
- ,device
- ,batch_size
- ,dataset
- ,weight
- , **kwargs):
- """ Run inference and evalaute the model performance """
-
- # PyTorch dataloader
- data_loader = DataLoader(
- dataset = dataset
- ,batch_size = batch_size
- ,shuffle= False
- ,pin_memory = torch.cuda.is_available()
- )
-
- # checkpointer, load last model in dir
- checkpointer = DetectronCheckpointer(model, save_dir = output_path, save_to_disk=False)
- checkpointer.load(weight)
- do_inference(model, data_loader, device, output_path)
-
-
-
-# Plot comparison
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path-list',
- '-l',
- required=True,
- help='Pass all output paths as arguments',
- cls=OptionEatAll,
- )
-@click.option(
- '--output-path',
- '-o',
- required=True,
- )
-@click.option(
- '--title',
- '-t',
- required=False,
- )
-@verbosity_option(cls=ResourceOption)
-def compare(output_path_list, output_path, title, **kwargs):
- """ Compares multiple metrics files that are stored in the format mymodel/results/Metrics.csv """
- logger.debug("Output paths: {}".format(output_path_list))
- logger.info('Plotting precision vs recall curves for {}'.format(output_path_list))
- fig = plot_overview(output_path_list,title)
- if not os.path.exists(output_path): os.makedirs(output_path)
- fig_filename = os.path.join(output_path, 'precision_recall_comparison.pdf')
- logger.info('saving {}'.format(fig_filename))
- fig.savefig(fig_filename)
-
-
-# Create grid table with results
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- )
-@verbosity_option(cls=ResourceOption)
-def gridtable(output_path, **kwargs):
- """ Creates an overview table in grid rst format for all Metrics.csv in the output_path
- tree structure:
- ├── DATABASE
- ├── MODEL
- ├── images
- └── results
+def save_sh_command(destfile):
+ """Records command-line to reproduce this experiment
+
+ This function can record the current command-line used to call the script
+ being run. It creates an executable ``bash`` script setting up the current
+ working directory and activating a conda environment, if needed. It
+ records further information on the date and time the script was run and the
+ version of the package.
+
+
+ Parameters
+ ----------
+
+ destfile : str
+ Path leading to the file where the commands to reproduce the current
+ run will be recorded. This file cannot be overwritten by this
+ function. If needed, you should check and remove an existing file
+ **before** calling this function.
+
"""
- logger.info('Creating grid for all results in {}'.format(output_path))
- create_overview_grid(output_path)
-
-
-# Create metrics viz
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--output-path',
- '-o',
- required=True,
- )
-@verbosity_option(cls=ResourceOption)
-def visualize(dataset, output_path, **kwargs):
- """ Creates the following visualizations of the probabilties output maps:
- overlayed: test images overlayed with prediction probabilities vessel tree
- tpfnfpviz: highlights true positives, false negatives and false positives
-
- Required tree structure:
- ├── DATABASE
- ├── MODEL
- ├── images
- └── results
+
+ if os.path.exists(destfile) and not overwrite:
+ logger.info(f"Not overwriting existing file '{destfile}'")
+ return
+
+ logger.info(f"Writing command-line for reproduction at '{destfile}'...")
+ os.makedirs(os.path.dirname(destfile), exist_ok=True)
+
+ with open(destfile, "wt") as f:
+ f.write("#!/usr/bin/env sh\n")
+ f.write(f"# date: {time.asctime()}\n")
+ version = pkg_resources.require("bob.ip.binseg")[0].version
+ f.write(f"# version: {version} (bob.ip.binseg)\n")
+ f.write(f"# platform: {sys.platform}\n")
+ f.write("\n")
+ args = []
+ for k in sys.argv:
+ if " " in k:
+ args.append(f'"{k}"')
+ else:
+ args.append(k)
+ if os.environ.get("CONDA_DEFAULT_ENV") is not None:
+ f.write(f"#conda activate {os.environ['CONDA_DEFAULT_ENV']}\n")
+ f.write(f"#cd {os.path.realpath(os.curdir)}\n")
+ f.write(" ".join(args) + "\n")
+ os.chmod(destfile, 0o755)
+
+
+def download_to_tempfile(url, progress=False):
+ """Downloads a file to a temporary named file and returns it
+
+ Parameters
+ ----------
+
+ url : str
+ The URL pointing to the file to download
+
+ progress : :py:class:`bool`, Optional
+ If a progress bar should be displayed for downloading the URL.
+
+
+ Returns
+ -------
+
+ f : tempfile.NamedTemporaryFile
+ A named temporary file that contains the downloaded URL
+
"""
- logger.info('Creating TP, FP, FN visualizations for {}'.format(output_path))
- metricsviz(dataset=dataset, output_path=output_path)
- logger.info('Creating overlay visualizations for {}'.format(output_path))
- overlay(dataset=dataset, output_path=output_path)
- logger.info('Saving transformed test images {}'.format(output_path))
- savetransformedtest(dataset=dataset, output_path=output_path)
-
-
-# SSLTrain
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- default="output",
- cls=ResourceOption
- )
-@click.option(
- '--model',
- '-m',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--optimizer',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--criterion',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--scheduler',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--pretrained-backbone',
- '-t',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--batch-size',
- '-b',
- required=True,
- default=2,
- cls=ResourceOption)
-@click.option(
- '--epochs',
- '-e',
- help='Number of epochs used for training',
- show_default=True,
- required=True,
- default=1000,
- cls=ResourceOption)
-@click.option(
- '--checkpoint-period',
- '-p',
- help='Number of epochs after which a checkpoint is saved',
- show_default=True,
- required=True,
- default=100,
- cls=ResourceOption)
-@click.option(
- '--device',
- '-d',
- help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
- show_default=True,
- required=True,
- default='cpu',
- cls=ResourceOption)
-@click.option(
- '--rampup',
- '-r',
- help='Ramp-up length in epochs',
- show_default=True,
- required=True,
- default='900',
- cls=ResourceOption)
-@click.option(
- '--seed',
- '-s',
- help='torch random seed',
- show_default=True,
- required=False,
- default=42,
- cls=ResourceOption)
-
-@verbosity_option(cls=ResourceOption)
-def ssltrain(model
- ,optimizer
- ,scheduler
- ,output_path
- ,epochs
- ,pretrained_backbone
- ,batch_size
- ,criterion
- ,dataset
- ,checkpoint_period
- ,device
- ,rampup
- ,seed
- ,**kwargs):
- """ Train a model """
-
- if not os.path.exists(output_path): os.makedirs(output_path)
- torch.manual_seed(seed)
- # PyTorch dataloader
- data_loader = DataLoader(
- dataset = dataset
- ,batch_size = batch_size
- ,shuffle= True
- ,pin_memory = torch.cuda.is_available()
- )
-
- # Checkpointer
- checkpointer = DetectronCheckpointer(model, optimizer, scheduler,save_dir = output_path, save_to_disk=True)
- arguments = {}
- arguments["epoch"] = 0
- extra_checkpoint_data = checkpointer.load(pretrained_backbone)
- arguments.update(extra_checkpoint_data)
- arguments["max_epoch"] = epochs
-
- # Train
- logger.info("Training for {} epochs".format(arguments["max_epoch"]))
- logger.info("Continuing from epoch {}".format(arguments["epoch"]))
- do_ssltrain(model
- , data_loader
- , optimizer
- , criterion
- , scheduler
- , checkpointer
- , checkpoint_period
- , device
- , arguments
- , output_path
- , rampup
- )
-
-# Apply image transforms to a folder containing images
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--source-path',
- '-s',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--target-path',
- '-t',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--transforms',
- '-a',
- required=True,
- cls=ResourceOption
- )
-
-@verbosity_option(cls=ResourceOption)
-def transformfolder(source_path ,target_path,transforms,**kwargs):
- logger.info('Applying transforms to images in {} and saving them to {}'.format(source_path, target_path))
- transfld(source_path,target_path,transforms)
-
-
-# Run inference and create predictions only (no ground truth available)
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- default="output",
- cls=ResourceOption
- )
-@click.option(
- '--model',
- '-m',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--batch-size',
- '-b',
- required=True,
- default=2,
- cls=ResourceOption)
-@click.option(
- '--device',
- '-d',
- help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
- show_default=True,
- required=True,
- default='cpu',
- cls=ResourceOption)
-@click.option(
- '--weight',
- '-w',
- help='Path or URL to pretrained model',
- required=False,
- default=None,
- cls=ResourceOption
- )
-@verbosity_option(cls=ResourceOption)
-def predict(model
- ,output_path
- ,device
- ,batch_size
- ,dataset
- ,weight
- , **kwargs):
- """ Run inference and evalaute the model performance """
-
- # PyTorch dataloader
- data_loader = DataLoader(
- dataset = dataset
- ,batch_size = batch_size
- ,shuffle= False
- ,pin_memory = torch.cuda.is_available()
- )
-
- # checkpointer, load last model in dir
- checkpointer = DetectronCheckpointer(model, save_dir = output_path, save_to_disk=False)
- checkpointer.load(weight)
- do_predict(model, data_loader, device, output_path)
-
- # Overlayed images
- overlay(dataset=dataset, output_path=output_path)
-
-
-
-# Evaluate only. Runs evaluation on predicted probability maps (--prediction-folder)
-@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
-@click.option(
- '--output-path',
- '-o',
- required=True,
- default="output",
- cls=ResourceOption
- )
-@click.option(
- '--prediction-folder',
- '-p',
- help = 'Path containing output probability maps',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--prediction-extension',
- '-x',
- help = 'Extension (e.g. ".png") for the prediction files',
- default=".png",
- required=False,
- cls=ResourceOption
- )
-@click.option(
- '--dataset',
- '-d',
- required=True,
- cls=ResourceOption
- )
-@click.option(
- '--title',
- required=False,
- cls=ResourceOption
- )
-@click.option(
- '--legend',
- cls=ResourceOption
- )
-
-@verbosity_option(cls=ResourceOption)
-def evalpred(
- output_path
- ,prediction_folder
- ,prediction_extension
- ,dataset
- ,title
- ,legend
- , **kwargs):
- """ Run inference and evalaute the model performance """
-
- # PyTorch dataloader
- data_loader = DataLoader(
- dataset = dataset
- ,batch_size = 1
- ,shuffle= False
- ,pin_memory = torch.cuda.is_available()
- )
-
- # Run eval
- do_eval(prediction_folder, data_loader, output_folder = output_path, title=title, legend=legend, prediction_extension=prediction_extension)
+ file_size = 0
+ response = urllib.request.urlopen(url)
+ meta = response.info()
+ if hasattr(meta, "getheaders"):
+ content_length = meta.getheaders("Content-Length")
+ else:
+ content_length = meta.get_all("Content-Length")
+ if content_length is not None and len(content_length) > 0:
+ file_size = int(content_length[0])
+ progress &= bool(file_size)
+
+ f = tempfile.NamedTemporaryFile()
+
+ with tqdm(total=file_size, disable=not progress) as pbar:
+ while True:
+ buffer = response.read(8192)
+ if len(buffer) == 0:
+ break
+ f.write(buffer)
+ pbar.update(len(buffer))
+
+ f.flush()
+ f.seek(0)
+ return f
+
+
+@with_plugins(pkg_resources.iter_entry_points("bob.ip.binseg.cli"))
+@click.group(cls=AliasedGroup)
+def binseg():
+ """Binary 2D Image Segmentation Benchmark commands."""
diff --git a/bob/ip/binseg/script/compare.py b/bob/ip/binseg/script/compare.py
new file mode 100644
index 0000000000000000000000000000000000000000..813a5cb8c392ceb2eb280d9e45a36752b01839b5
--- /dev/null
+++ b/bob/ip/binseg/script/compare.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import click
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ AliasedGroup,
+)
+
+import pandas
+import tabulate
+
+from ..utils.plot import precision_recall_f1iso
+from ..utils.table import performance_table
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def _validate_threshold(t, dataset):
+ """Validates the user threshold selection. Returns parsed threshold."""
+
+ if t is None:
+ return t
+
+ try:
+ # we try to convert it to float first
+ t = float(t)
+ if t < 0.0 or t > 1.0:
+ raise ValueError("Float thresholds must be within range [0.0, 1.0]")
+ except ValueError:
+ # it is a bit of text - assert dataset with name is available
+ if not isinstance(dataset, dict):
+ raise ValueError(
+ "Threshold should be a floating-point number "
+ "if your provide only a single dataset for evaluation"
+ )
+ if t not in dataset:
+ raise ValueError(
+ f"Text thresholds should match dataset names, "
+ f"but {t} is not available among the datasets provided ("
+ f"({', '.join(dataset.keys())})"
+ )
+
+ return t
+
+
+def _load(data, threshold=None):
+ """Plots comparison chart of all evaluated models
+
+ Parameters
+ ----------
+
+ data : dict
+ A dict in which keys are the names of the systems and the values are
+ paths to ``metrics.csv`` style files.
+
+ threshold : :py:class:`float`, :py:class:`str`, Optional
+ A value indicating which threshold to choose for selecting a "F1-score"
+ If set to ``None``, then use the maximum F1-score on that metrics file.
+ If set to a floating-point value, then use the F1-score that is
+ obtained on that particular threshold. If set to a string, it should
+ match one of the keys in ``data``. It then first calculate the
+ threshold reaching the maximum F1-score on that particular dataset and
+ then applies that threshold to all other sets.
+
+
+ Returns
+ -------
+
+ data : dict
+ A dict in which keys are the names of the systems and the values are
+ dictionaries that contain two keys:
+
+ * ``df``: A :py:class:`pandas.DataFrame` with the metrics data loaded
+ to
+ * ``threshold``: A threshold to be used for summarization, depending on
+ the ``threshold`` parameter set on the input
+
+ """
+
+ if isinstance(threshold, str):
+ logger.info(f"Calculating threshold from maximum F1-score at "
+ f"'{threshold}' dataset...")
+ metrics_path = data[threshold]
+ df = pandas.read_csv(metrics_path)
+
+ maxf1 = df.f1_score.max()
+ use_threshold = df.threshold[df.f1_score.idxmax()]
+ logger.info(f"Dataset '*': threshold = {use_threshold:.3f}'")
+
+ elif isinstance(threshold, float):
+ use_threshold = threshold
+ logger.info(f"Dataset '*': threshold = {use_threshold:.3f}'")
+
+ names = []
+ dfs = []
+ thresholds = []
+
+ # loads all data
+ retval = {}
+ for name, metrics_path in data.items():
+
+ logger.info(f"Loading metrics from {metrics_path}...")
+ df = pandas.read_csv(metrics_path)
+
+ if threshold is None:
+ use_threshold = df.threshold[df.f1_score.idxmax()]
+ logger.info(f"Dataset '{name}': threshold = {use_threshold:.3f}'")
+
+ retval[name] = dict(df=df, threshold=use_threshold)
+
+ return retval
+
+
+@click.command(
+ epilog="""Examples:
+
+\b
+ 1. Compares system A and B, with their own pre-computed metric files:
+\b
+ $ bob binseg compare -vv A path/to/A/metrics.csv B path/to/B/metrics.csv
+""",
+)
+@click.argument(
+ 'label_path',
+ nargs=-1,
+ )
+@click.option(
+ "--output-figure",
+ "-f",
+ help="Path where write the output figure (any extension supported by "
+ "matplotlib is possible). If not provided, does not produce a figure.",
+ required=False,
+ default=None,
+ type=click.Path(dir_okay=False, file_okay=True),
+)
+@click.option(
+ "--table-format",
+ "-T",
+ help="The format to use for the comparison table",
+ show_default=True,
+ required=True,
+ default="rst",
+ type=click.Choice(tabulate.tabulate_formats),
+)
+@click.option(
+ "--output-table",
+ "-u",
+ help="Path where write the output table. If not provided, does not write "
+ "write a table to file, only to stdout.",
+ required=False,
+ default=None,
+ type=click.Path(dir_okay=False, file_okay=True),
+)
+@click.option(
+ "--threshold",
+ "-t",
+ help="This number is used to select which F1-score to use for "
+ "representing a system performance. If not set, we report the maximum "
+ "F1-score in the set, which is equivalent to threshold selection a "
+ "posteriori (biased estimator). You can either set this value to a "
+ "floating-point number in the range [0.0, 1.0], or to a string, naming "
+ "one of the systems which will be used to calculate the threshold "
+ "leading to the maximum F1-score and then applied to all other sets.",
+ default=None,
+ show_default=False,
+ required=False,
+)
+@verbosity_option()
+def compare(label_path, output_figure, table_format, output_table, threshold,
+ **kwargs):
+ """Compares multiple systems together"""
+
+ # hack to get a dictionary from arguments passed to input
+ if len(label_path) % 2 != 0:
+ raise click.ClickException("Input label-paths should be doubles"
+ " composed of name-path entries")
+ data = dict(zip(label_path[::2], label_path[1::2]))
+
+ threshold = _validate_threshold(threshold, data)
+
+ # load all data metrics
+ data = _load(data, threshold=threshold)
+
+ if output_figure is not None:
+ output_figure = os.path.realpath(output_figure)
+ logger.info(f"Creating and saving plot at {output_figure}...")
+ os.makedirs(os.path.dirname(output_figure), exist_ok=True)
+ fig = precision_recall_f1iso(data, confidence=True)
+ fig.savefig(output_figure)
+
+ logger.info("Tabulating performance summary...")
+ table = performance_table(data, table_format)
+ click.echo(table)
+ if output_table is not None:
+ output_table = os.path.realpath(output_table)
+ logger.info(f"Saving table at {output_table}...")
+ os.makedirs(os.path.dirname(output_table), exist_ok=True)
+ with open(output_table, "wt") as f:
+ f.write(table)
diff --git a/bob/ip/binseg/script/config.py b/bob/ip/binseg/script/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ab6b4ca71493a6d20903cfcbd53fd9a1365bb9e
--- /dev/null
+++ b/bob/ip/binseg/script/config.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import shutil
+import inspect
+
+import click
+import pkg_resources
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ AliasedGroup,
+)
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@click.group(cls=AliasedGroup)
+def config():
+ """Commands for listing, describing and copying configuration resources"""
+ pass
+
+
+@config.command(
+ epilog="""
+\b
+Examples:
+
+\b
+ 1. Lists all configuration resources (type: bob.ip.binseg.config) installed:
+
+\b
+ $ bob binseg config list
+
+
+\b
+ 2. Lists all configuration resources and their descriptions (notice this may
+ be slow as it needs to load all modules once):
+
+\b
+ $ bob binseg config list -v
+
+"""
+)
+@verbosity_option()
+def list(verbose):
+ """Lists configuration files installed"""
+
+ entry_points = pkg_resources.iter_entry_points("bob.ip.binseg.config")
+ entry_points = dict([(k.name, k) for k in entry_points])
+
+ # all modules with configuration resources
+ modules = set(
+ k.module_name.rsplit(".", 1)[0] for k in entry_points.values()
+ )
+ keep_modules = []
+ for k in sorted(modules):
+ if k not in keep_modules and \
+ not any(k.startswith(l) for l in keep_modules):
+ keep_modules.append(k)
+ modules = keep_modules
+
+ # sort data entries by originating module
+ entry_points_by_module = {}
+ for k in modules:
+ entry_points_by_module[k] = {}
+ for name, ep in entry_points.items():
+ if ep.module_name.startswith(k):
+ entry_points_by_module[k][name] = ep
+
+ for config_type in sorted(entry_points_by_module):
+
+ # calculates the longest config name so we offset the printing
+ longest_name_length = max(
+ len(k) for k in entry_points_by_module[config_type].keys()
+ )
+
+ # set-up printing options
+ print_string = " %%-%ds %%s" % (longest_name_length,)
+ # 79 - 4 spaces = 75 (see string above)
+ description_leftover = 75 - longest_name_length
+
+ print("module: %s" % (config_type,))
+ for name in sorted(entry_points_by_module[config_type]):
+ ep = entry_points[name]
+
+ if verbose >= 1:
+ module = ep.load()
+ doc = inspect.getdoc(module)
+ if doc is not None:
+ summary = doc.split("\n\n")[0]
+ else:
+ summary = "<DOCSTRING NOT AVAILABLE>"
+ else:
+ summary = ""
+
+ summary = (
+ (summary[: (description_leftover - 3)] + "...")
+ if len(summary) > (description_leftover - 3)
+ else summary
+ )
+
+ print(print_string % (name, summary))
+
+
+@config.command(
+ epilog="""
+\b
+Examples:
+
+\b
+ 1. Describes the DRIVE (training) dataset configuration:
+
+\b
+ $ bob binseg config describe drive
+
+
+\b
+ 2. Describes the DRIVE (training) dataset configuration and lists its
+ contents:
+
+\b
+ $ bob binseg config describe drive -v
+
+"""
+)
+@click.argument(
+ "name", required=True, nargs=-1,
+)
+@verbosity_option()
+def describe(name, verbose):
+ """Describes a specific configuration file"""
+
+ entry_points = pkg_resources.iter_entry_points("bob.ip.binseg.config")
+ entry_points = dict([(k.name, k) for k in entry_points])
+
+ for k in name:
+ if k not in entry_points:
+ logger.error("Cannot find configuration resource '%s'", k)
+ continue
+ ep = entry_points[k]
+ print("Configuration: %s" % (ep.name,))
+ print("Python Module: %s" % (ep.module_name,))
+ print("")
+ mod = ep.load()
+
+ if verbose >= 1:
+ fname = inspect.getfile(mod)
+ print("Contents:")
+ with open(fname, "r") as f:
+ print(f.read())
+ else: #only output documentation
+ print("Documentation:")
+ print(inspect.getdoc(mod))
+
+
+@config.command(
+ epilog="""
+\b
+Examples:
+
+\b
+ 1. Makes a copy of one of the stock configuration files locally, so it can be
+ adapted:
+
+\b
+ $ bob binseg config copy drive -vvv newdataset.py
+
+
+"""
+)
+@click.argument(
+ "source", required=True, nargs=1,
+)
+@click.argument(
+ "destination", required=True, nargs=1,
+)
+@verbosity_option()
+def copy(source, destination, verbose):
+ """Copies a specific configuration resource so it can be modified locally"""
+
+ entry_points = pkg_resources.iter_entry_points("bob.ip.binseg.config")
+ entry_points = dict([(k.name, k) for k in entry_points])
+
+ if source not in entry_points:
+ logger.error("Cannot find configuration resource '%s'", source)
+ return 1
+ ep = entry_points[source]
+ mod = ep.load()
+ src_name = inspect.getfile(mod)
+ logger.info('cp %s -> %s' % (src_name, destination))
+ shutil.copyfile(src_name, destination)
diff --git a/bob/ip/binseg/script/dataset.py b/bob/ip/binseg/script/dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2eb52db9cf95ad92423e63713a5922680c972ba
--- /dev/null
+++ b/bob/ip/binseg/script/dataset.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import pkg_resources
+import importlib
+import click
+
+from bob.extension import rc
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ AliasedGroup,
+)
+
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def _get_supported_datasets():
+ """Returns a list of supported dataset names
+ """
+
+ basedir = pkg_resources.resource_filename(__name__, '')
+ basedir = os.path.join(os.path.dirname(basedir), 'data')
+
+ retval = []
+ for k in os.listdir(basedir):
+ candidate = os.path.join(basedir, k)
+ if os.path.isdir(candidate) and '__init__.py' in os.listdir(candidate):
+ retval.append(k)
+ return retval
+
+def _get_installed_datasets():
+ """Returns a list of installed datasets as regular expressions
+
+ * group(0): the name of the key for the dataset directory
+ * group("name"): the short name for the dataset
+
+ """
+
+ import re
+ dataset_re = re.compile(r'^bob\.ip\.binseg\.(?P<name>[^\.]+)\.datadir$')
+ return [dataset_re.match(k) for k in rc.keys() if dataset_re.match(k)]
+
+
+@click.group(cls=AliasedGroup)
+def dataset():
+ """Commands for listing and verifying datasets"""
+ pass
+
+
+@dataset.command(
+ epilog="""Examples:
+
+\b
+ 1. To install a dataset, set up its data directory ("datadir"). For
+ example, to setup access to DRIVE files you downloaded locally at
+ the directory "/path/to/drive/files", do the following:
+\b
+ $ bob config set "bob.ip.binseg.drive.datadir" "/path/to/drive/files"
+
+ Notice this setting **is** case-sensitive.
+
+ 2. List all raw datasets supported (and configured):
+
+ $ bob binseg dataset list
+
+""",
+)
+@verbosity_option()
+def list(**kwargs):
+ """Lists all supported and configured datasets"""
+
+ supported = _get_supported_datasets()
+ installed = _get_installed_datasets()
+ installed = dict((k.group("name"), k.group(0)) for k in installed)
+
+ click.echo("Supported datasets:")
+ for k in supported:
+ if k in installed:
+ click.echo(f"- {k}: {installed[k]} = \"{rc.get(installed[k])}\"")
+ else:
+ click.echo(f"* {k}: bob.ip.binseg.{k}.datadir (not set)")
+
+
+@dataset.command(
+ epilog="""Examples:
+
+ 1. Check if all files of the DRIVE dataset can be loaded:
+
+ $ bob binseg dataset check -vv drive
+
+ 2. Check if all files of multiple installed datasets can be loaded:
+
+ $ bob binseg dataset check -vv drive stare
+
+ 3. Check if all files of all installed datasets can be loaded:
+
+ $ bob binseg dataset check
+""",
+)
+@click.argument(
+ 'dataset',
+ nargs=-1,
+ )
+@click.option(
+ "--limit",
+ "-l",
+ help="Limit check to the first N samples in each dataset, making the "
+ "check sensibly faster. Set it to zero to check everything.",
+ required=True,
+ type=click.IntRange(0),
+ default=0,
+)
+@verbosity_option()
+def check(dataset, limit, **kwargs):
+ """Checks file access on one or more datasets"""
+
+ to_check = _get_installed_datasets()
+
+ if dataset: #check only some
+ to_check = [k for k in to_check if k.group("name") in dataset]
+
+ if not to_check:
+ click.echo("No configured datasets matching specifications")
+ click.echo("Try bob binseg dataset list --help to get help in "
+ "configuring a dataset")
+ else:
+ errors = 0
+ for k in to_check:
+ click.echo(f"Checking \"{k.group('name')}\" dataset...")
+ module = importlib.import_module(f"...data.{k.group('name')}",
+ __name__)
+ errors += module.dataset.check(limit)
+ if not errors:
+ click.echo(f"No errors reported")
diff --git a/bob/ip/binseg/script/evaluate.py b/bob/ip/binseg/script/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a4b33d1a7a44991bbc827dc25f83f127fd47875
--- /dev/null
+++ b/bob/ip/binseg/script/evaluate.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import click
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ ConfigCommand,
+ ResourceOption,
+)
+
+from ..engine.evaluator import run, compare_annotators
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def _validate_threshold(t, dataset):
+ """Validates the user threshold selection. Returns parsed threshold."""
+
+ if t is None:
+ return 0.5
+
+ try:
+ # we try to convert it to float first
+ t = float(t)
+ if t < 0.0 or t > 1.0:
+ raise ValueError("Float thresholds must be within range [0.0, 1.0]")
+ except ValueError:
+ # it is a bit of text - assert dataset with name is available
+ if not isinstance(dataset, dict):
+ raise ValueError(
+ "Threshold should be a floating-point number "
+ "if your provide only a single dataset for evaluation"
+ )
+ if t not in dataset:
+ raise ValueError(
+ f"Text thresholds should match dataset names, "
+ f"but {t} is not available among the datasets provided ("
+ f"({', '.join(dataset.keys())})"
+ )
+
+ return t
+
+
+@click.command(
+ entry_point_group="bob.ip.binseg.config",
+ cls=ConfigCommand,
+ epilog="""Examples:
+
+\b
+ 1. Runs evaluation on an existing dataset configuration:
+\b
+ $ bob binseg evaluate -vv drive --predictions-folder=path/to/predictions --output-folder=path/to/results
+\b
+ 2. To run evaluation on a folder with your own images and annotations, you
+ must first specify resizing, cropping, etc, so that the image can be
+ correctly input to the model. Failing to do so will likely result in
+ poor performance. To figure out such specifications, you must consult
+ the dataset configuration used for **training** the provided model.
+ Once you figured this out, do the following:
+\b
+ $ bob binseg config copy csv-dataset-example mydataset.py
+ # modify "mydataset.py" to your liking
+ $ bob binseg evaluate -vv mydataset.py --predictions-folder=path/to/predictions --output-folder=path/to/results
+""",
+)
+@click.option(
+ "--output-folder",
+ "-o",
+ help="Path where to store the analysis result (created if does not exist)",
+ required=True,
+ default="results",
+ type=click.Path(),
+ cls=ResourceOption,
+)
+@click.option(
+ "--predictions-folder",
+ "-p",
+ help="Path where predictions are currently stored",
+ required=True,
+ type=click.Path(exists=True, file_okay=False, dir_okay=True),
+ cls=ResourceOption,
+)
+@click.option(
+ "--dataset",
+ "-d",
+ help="A torch.utils.data.dataset.Dataset instance implementing a dataset "
+ "to be used for evaluation purposes, possibly including all pre-processing "
+ "pipelines required or, optionally, a dictionary mapping string keys to "
+ "torch.utils.data.dataset.Dataset instances. All keys that do not start "
+ "with an underscore (_) will be processed.",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--second-annotator",
+ "-S",
+ help="A dataset or dictionary, like in --dataset, with the same "
+ "sample keys, but with annotations from a different annotator that is "
+ "going to be compared to the one in --dataset. The same rules regarding "
+ "dataset naming conventions apply",
+ required=False,
+ default=None,
+ cls=ResourceOption,
+ show_default=True,
+)
+@click.option(
+ "--overlayed",
+ "-O",
+ help="Creates overlayed representations of the output probability maps, "
+ "similar to --overlayed in prediction-mode, except it includes "
+ "distinctive colours for true and false positives and false negatives. "
+ "If not set, or empty then do **NOT** output overlayed images. "
+ "Otherwise, the parameter represents the name of a folder where to "
+ "store those",
+ show_default=True,
+ default=None,
+ required=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--threshold",
+ "-t",
+ help="This number is used to define positives and negatives from "
+ "probability maps, and report F1-scores (a priori). It "
+ "should either come from the training set or a separate validation set "
+ "to avoid biasing the analysis. Optionally, if you provide a multi-set "
+ "dataset as input, this may also be the name of an existing set from "
+ "which the threshold will be estimated (highest F1-score) and then "
+ "applied to the subsequent sets. This number is also used to print "
+ "the test set F1-score a priori performance (default: 0.5)",
+ default=None,
+ show_default=False,
+ required=False,
+ cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+def evaluate(
+ output_folder,
+ predictions_folder,
+ dataset,
+ second_annotator,
+ overlayed,
+ threshold,
+ **kwargs,
+):
+ """Evaluates an FCN on a binary segmentation task.
+ """
+
+ threshold = _validate_threshold(threshold, dataset)
+
+ if not isinstance(dataset, dict):
+ dataset = {"test": dataset}
+
+ if second_annotator is None:
+ second_annotator = {}
+ elif not isinstance(second_annotator, dict):
+ second_annotator = {"test": second_annotator}
+ #else, second_annotator must be a dict
+
+ if isinstance(threshold, str):
+ # first run evaluation for reference dataset, do not save overlays
+ logger.info(f"Evaluating threshold on '{threshold}' set")
+ threshold = run(dataset[threshold], threshold, predictions_folder)
+ logger.info(f"Set --threshold={threshold:.5f}")
+
+ # now run with the
+ for k, v in dataset.items():
+ if k.startswith("_"):
+ logger.info(f"Skipping dataset '{k}' (not to be evaluated)")
+ continue
+ logger.info(f"Analyzing '{k}' set...")
+ run(v, k, predictions_folder, output_folder, overlayed, threshold)
+ second = second_annotator.get(k)
+ if second is not None:
+ compare_annotators(v, second, k, output_folder, overlayed)
diff --git a/bob/ip/binseg/script/experiment.py b/bob/ip/binseg/script/experiment.py
new file mode 100644
index 0000000000000000000000000000000000000000..cbbfd56f0754327b6bb93abde03b4718c387d930
--- /dev/null
+++ b/bob/ip/binseg/script/experiment.py
@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import shutil
+
+import click
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ ConfigCommand,
+ ResourceOption,
+)
+
+from .binseg import save_sh_command
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+@click.command(
+ entry_point_group="bob.ip.binseg.config",
+ cls=ConfigCommand,
+ epilog="""Examples:
+
+\b
+ 1. Trains an M2U-Net model (VGG-16 backbone) with DRIVE (vessel
+ segmentation), on the CPU, for only two epochs, then runs inference and
+ evaluation on stock datasets, report performance as a table and a figure:
+
+ $ bob binseg experiment -vv m2unet drive --epochs=2
+
+""",
+)
+@click.option(
+ "--output-folder",
+ "-o",
+ help="Path where to store experiment outputs (created if does not exist)",
+ required=True,
+ type=click.Path(),
+ default="results",
+ cls=ResourceOption,
+)
+@click.option(
+ "--model",
+ "-m",
+ help="A torch.nn.Module instance implementing the network to be trained, and then evaluated",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--dataset",
+ "-d",
+ help="A dictionary mapping string keys to "
+ "bob.ip.binseg.data.utils.SampleList2TorchDataset's. At least one key "
+ "named 'train' must be available. This dataset will be used for training "
+ "the network model. All other datasets will be used for prediction and "
+ "evaluation. Dataset descriptions include all required pre-processing, "
+ "including eventual data augmentation, which may be eventually excluded "
+ "for prediction and evaluation purposes",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--second-annotator",
+ "-S",
+ help="A dataset or dictionary, like in --dataset, with the same "
+ "sample keys, but with annotations from a different annotator that is "
+ "going to be compared to the one in --dataset",
+ required=False,
+ default=None,
+ cls=ResourceOption,
+ show_default=True,
+)
+@click.option(
+ "--optimizer",
+ help="A torch.optim.Optimizer that will be used to train the network",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--criterion",
+ help="A loss function to compute the FCN error for every sample "
+ "respecting the PyTorch API for loss functions (see torch.nn.modules.loss)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--scheduler",
+ help="A learning rate scheduler that drives changes in the learning "
+ "rate depending on the FCN state (see torch.optim.lr_scheduler)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--pretrained-backbone",
+ "-t",
+ help="URL of a pre-trained model file that will be used to preset "
+ "FCN weights (where relevant) before training starts "
+ "(e.g. vgg16, mobilenetv2)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--batch-size",
+ "-b",
+ help="Number of samples in every batch (this parameter affects "
+ "memory requirements for the network). If the number of samples in "
+ "the batch is larger than the total number of samples available for "
+ "training, this value is truncated. If this number is smaller, then "
+ "batches of the specified size are created and fed to the network "
+ "until there are no more new samples to feed (epoch is finished). "
+ "If the total number of training samples is not a multiple of the "
+ "batch-size, the last batch will be smaller than the first, unless "
+ "--drop-incomplete--batch is set, in which case this batch is not used.",
+ required=True,
+ show_default=True,
+ default=2,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--drop-incomplete-batch/--no-drop-incomplete-batch",
+ "-D",
+ help="If set, then may drop the last batch in an epoch, in case it is "
+ "incomplete. If you set this option, you should also consider "
+ "increasing the total number of epochs of training, as the total number "
+ "of training steps may be reduced",
+ required=True,
+ show_default=True,
+ default=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--epochs",
+ "-e",
+ help="Number of epochs (complete training set passes) to train for",
+ show_default=True,
+ required=True,
+ default=1000,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--checkpoint-period",
+ "-p",
+ help="Number of epochs after which a checkpoint is saved. "
+ "A value of zero will disable check-pointing. If checkpointing is "
+ "enabled and training stops, it is automatically resumed from the "
+ "last saved checkpoint if training is restarted with the same "
+ "configuration.",
+ show_default=True,
+ required=True,
+ default=0,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@click.option(
+ "--device",
+ "-d",
+ help='A string indicating the device to use (e.g. "cpu" or "cuda:0")',
+ show_default=True,
+ required=True,
+ default="cpu",
+ cls=ResourceOption,
+)
+@click.option(
+ "--seed",
+ "-s",
+ help="Seed to use for the random number generator",
+ show_default=True,
+ required=False,
+ default=42,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@click.option(
+ "--ssl/--no-ssl",
+ help="Switch ON/OFF semi-supervised training mode",
+ show_default=True,
+ required=True,
+ default=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--rampup",
+ "-r",
+ help="Ramp-up length in epochs (for SSL training only)",
+ show_default=True,
+ required=True,
+ default=900,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@click.option(
+ "--overlayed/--no-overlayed",
+ "-O",
+ help="Creates overlayed representations of the output probability maps, "
+ "similar to --overlayed in prediction-mode, except it includes "
+ "distinctive colours for true and false positives and false negatives. "
+ "If not set, or empty then do **NOT** output overlayed images.",
+ show_default=True,
+ default=False,
+ required=False,
+ cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+@click.pass_context
+def experiment(
+ ctx,
+ model,
+ optimizer,
+ scheduler,
+ output_folder,
+ epochs,
+ pretrained_backbone,
+ batch_size,
+ drop_incomplete_batch,
+ criterion,
+ dataset,
+ second_annotator,
+ checkpoint_period,
+ device,
+ seed,
+ ssl,
+ rampup,
+ overlayed,
+ verbose,
+ **kwargs,
+):
+ """Runs a complete experiment, from training, to prediction and evaluation
+
+ This script is just a wrapper around the individual scripts for training,
+ running prediction, evaluating and comparing FCN model performance. It
+ organises the output in a preset way::
+
+\b
+ └─ <output-folder>/
+ ├── model/ #the generated model will be here
+ ├── predictions/ #the prediction outputs for the train/test set
+ ├── overlayed/ #the overlayed outputs for the train/test set
+ ├── predictions/ #predictions overlayed on the input images
+ ├── analysis/ #predictions overlayed on the input images
+ ├ #including analysis of false positives, negatives
+ ├ #and true positives
+ └── second-annotator/ #if set, store overlayed images for the
+ #second annotator here
+ └── analysis / #the outputs of the analysis of both train/test sets
+ #includes second-annotator "metrics" as well, if
+ # configured
+
+ Training is performed for a configurable number of epochs, and generates at
+ least a final_model.pth. It may also generate a number of intermediate
+ checkpoints. Checkpoints are model files (.pth files) that are stored
+ during the training and useful to resume the procedure in case it stops
+ abruptly.
+
+ N.B.: The tool is designed to prevent analysis bias and allows one to
+ provide separate subsets for training and evaluation. Instead of using
+ simple datasets, datasets for full experiment running should be
+ dictionaries with specific subset names:
+
+ * ``__train__``: dataset used for training, prioritarily. It is typically
+ the dataset containing data augmentation pipelines.
+ * ``train`` (optional): a copy of the ``__train__`` dataset, without data
+ augmentation, that will be evaluated alongside other sets available
+ * ``*``: any other name, not starting with an underscore character (``_``),
+ will be considered a test set for evaluation.
+
+ N.B.2: The threshold used for calculating the F1-score on the test set, or
+ overlay analysis (false positives, negatives and true positives overprinted
+ on the original image) also follows the logic above.
+ """
+
+ command_sh = os.path.join(output_folder, "command.sh")
+ if os.path.exists(command_sh):
+ backup = command_sh + '~'
+ if os.path.exists(backup):
+ os.unlink(backup)
+ shutil.move(command_sh, backup)
+ save_sh_command(command_sh)
+
+ ## Training
+ logger.info("Started training")
+
+ from .train import train
+
+ train_output_folder = os.path.join(output_folder, "model")
+
+ ctx.invoke(
+ train,
+ model=model,
+ optimizer=optimizer,
+ scheduler=scheduler,
+ output_folder=train_output_folder,
+ epochs=epochs,
+ pretrained_backbone=pretrained_backbone,
+ batch_size=batch_size,
+ drop_incomplete_batch=drop_incomplete_batch,
+ criterion=criterion,
+ dataset=dataset,
+ checkpoint_period=checkpoint_period,
+ device=device,
+ seed=seed,
+ ssl=ssl,
+ rampup=rampup,
+ verbose=verbose,
+ )
+ logger.info("Ended training")
+
+ from .analyze import analyze
+
+ model_file = os.path.join(train_output_folder, "model_final.pth")
+
+ ctx.invoke(
+ analyze,
+ model=model,
+ output_folder=output_folder,
+ batch_size=batch_size,
+ dataset=dataset,
+ second_annotator=second_annotator,
+ device=device,
+ overlayed=overlayed,
+ weight=model_file,
+ verbose=verbose,
+ )
diff --git a/bob/ip/binseg/script/predict.py b/bob/ip/binseg/script/predict.py
new file mode 100644
index 0000000000000000000000000000000000000000..14c9cd7495d05aff04a6edd8f3e85be9ef1b6129
--- /dev/null
+++ b/bob/ip/binseg/script/predict.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import tempfile
+
+import click
+import torch
+from torch.utils.data import DataLoader
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ ConfigCommand,
+ ResourceOption,
+)
+
+from ..engine.predictor import run
+from ..utils.checkpointer import DetectronCheckpointer
+
+from .binseg import download_to_tempfile
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@click.command(
+ entry_point_group="bob.ip.binseg.config",
+ cls=ConfigCommand,
+ epilog="""Examples:
+
+\b
+ 1. Runs prediction on an existing dataset configuration:
+\b
+ $ bob binseg predict -vv m2unet drive --weight=path/to/model_final.pth --output-folder=path/to/predictions
+\b
+ 2. To run prediction on a folder with your own images, you must first
+ specify resizing, cropping, etc, so that the image can be correctly
+ input to the model. Failing to do so will likely result in poor
+ performance. To figure out such specifications, you must consult the
+ dataset configuration used for **training** the provided model. Once
+ you figured this out, do the following:
+\b
+ $ bob binseg config copy csv-dataset-example mydataset.py
+ # modify "mydataset.py" to include the base path and required transforms
+ $ bob binseg predict -vv m2unet mydataset.py --weight=path/to/model_final.pth --output-folder=path/to/predictions
+""",
+)
+@click.option(
+ "--output-folder",
+ "-o",
+ help="Path where to store the predictions (created if does not exist)",
+ required=True,
+ default="results",
+ cls=ResourceOption,
+ type=click.Path(),
+)
+@click.option(
+ "--model",
+ "-m",
+ help="A torch.nn.Module instance implementing the network to be evaluated",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--dataset",
+ "-d",
+ help="A torch.utils.data.dataset.Dataset instance implementing a dataset "
+ "to be used for running prediction, possibly including all pre-processing "
+ "pipelines required or, optionally, a dictionary mapping string keys to "
+ "torch.utils.data.dataset.Dataset instances. All keys that do not start "
+ "with an underscore (_) will be processed.",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--batch-size",
+ "-b",
+ help="Number of samples in every batch (this parameter affects memory requirements for the network)",
+ required=True,
+ show_default=True,
+ default=1,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--device",
+ "-d",
+ help='A string indicating the device to use (e.g. "cpu" or "cuda:0")',
+ show_default=True,
+ required=True,
+ default="cpu",
+ cls=ResourceOption,
+)
+@click.option(
+ "--weight",
+ "-w",
+ help="Path or URL to pretrained model file (.pth extension)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--overlayed",
+ "-O",
+ help="Creates overlayed representations of the output probability maps on "
+ "top of input images (store results as PNG files). If not set, or empty "
+ "then do **NOT** output overlayed images. Otherwise, the parameter "
+ "represents the name of a folder where to store those",
+ show_default=True,
+ default=None,
+ required=False,
+ cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+def predict(output_folder, model, dataset, batch_size, device, weight,
+ overlayed, **kwargs):
+ """Predicts vessel map (probabilities) on input images"""
+
+ dataset = dataset if isinstance(dataset, dict) else dict(test=dataset)
+
+ if weight.startswith("http"):
+ logger.info(f"Temporarily downloading '{weight}'...")
+ f = download_to_tempfile(weight, progress=True)
+ weight_fullpath = os.path.abspath(f.name)
+ else:
+ weight_fullpath = os.path.abspath(weight)
+
+ weight_path = os.path.dirname(weight_fullpath)
+ weight_name = os.path.basename(weight_fullpath)
+ checkpointer = DetectronCheckpointer(model, save_dir=weight_path,
+ save_to_disk=False)
+ checkpointer.load(weight_name)
+
+ # clean-up the overlayed path
+ if overlayed is not None:
+ overlayed = overlayed.strip()
+
+ for k,v in dataset.items():
+
+ if k.startswith("_"):
+ logger.info(f"Skipping dataset '{k}' (not to be evaluated)")
+ continue
+
+ logger.info(f"Running inference on '{k}' set...")
+
+ data_loader = DataLoader(
+ dataset=v,
+ batch_size=batch_size,
+ shuffle=False,
+ pin_memory=torch.cuda.is_available(),
+ )
+ run(model, data_loader, device, output_folder, overlayed)
diff --git a/bob/ip/binseg/script/train.py b/bob/ip/binseg/script/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..3076aae4c9796fedb8ed009aea4ee6afb89edc85
--- /dev/null
+++ b/bob/ip/binseg/script/train.py
@@ -0,0 +1,289 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+
+import click
+import torch
+from torch.utils.data import DataLoader
+
+from bob.extension.scripts.click_helper import (
+ verbosity_option,
+ ConfigCommand,
+ ResourceOption,
+)
+
+from ..utils.checkpointer import DetectronCheckpointer
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@click.command(
+ entry_point_group="bob.ip.binseg.config",
+ cls=ConfigCommand,
+ epilog="""Examples:
+
+\b
+ 1. Trains a U-Net model (VGG-16 backbone) with DRIVE (vessel segmentation),
+ on a GPU (``cuda:0``):
+
+ $ bob binseg train -vv unet drive --batch-size=4 --device="cuda:0"
+
+ 2. Trains a HED model with HRF on a GPU (``cuda:0``):
+
+ $ bob binseg train -vv hed hrf --batch-size=8 --device="cuda:0"
+
+ 3. Trains a M2U-Net model on the COVD-DRIVE dataset on the CPU:
+
+ $ bob binseg train -vv m2unet covd-drive --batch-size=8
+
+ 4. Trains a DRIU model with SSL on the COVD-HRF dataset on the CPU:
+
+ $ bob binseg train -vv --ssl driu-ssl covd-drive-ssl --batch-size=1
+
+""",
+)
+@click.option(
+ "--output-folder",
+ "-o",
+ help="Path where to store the generated model (created if does not exist)",
+ required=True,
+ type=click.Path(),
+ default="results",
+ cls=ResourceOption,
+)
+@click.option(
+ "--model",
+ "-m",
+ help="A torch.nn.Module instance implementing the network to be trained",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--dataset",
+ "-d",
+ help="A torch.utils.data.dataset.Dataset instance implementing a dataset "
+ "to be used for training the model, possibly including all pre-processing "
+ "pipelines required or, optionally, a dictionary mapping string keys to "
+ "torch.utils.data.dataset.Dataset instances. At least one key "
+ "named ``train`` must be available. This dataset will be used for "
+ "training the network model. The dataset description must include all "
+ "required pre-processing, including eventual data augmentation. If a "
+ "dataset named ``__train__`` is available, it is used prioritarily for "
+ "training instead of ``train``.",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--optimizer",
+ help="A torch.optim.Optimizer that will be used to train the network",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--criterion",
+ help="A loss function to compute the FCN error for every sample "
+ "respecting the PyTorch API for loss functions (see torch.nn.modules.loss)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--scheduler",
+ help="A learning rate scheduler that drives changes in the learning "
+ "rate depending on the FCN state (see torch.optim.lr_scheduler)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--pretrained-backbone",
+ "-t",
+ help="URL of a pre-trained model file that will be used to preset "
+ "FCN weights (where relevant) before training starts "
+ "(e.g. vgg16, mobilenetv2)",
+ required=True,
+ cls=ResourceOption,
+)
+@click.option(
+ "--batch-size",
+ "-b",
+ help="Number of samples in every batch (this parameter affects "
+ "memory requirements for the network). If the number of samples in "
+ "the batch is larger than the total number of samples available for "
+ "training, this value is truncated. If this number is smaller, then "
+ "batches of the specified size are created and fed to the network "
+ "until there are no more new samples to feed (epoch is finished). "
+ "If the total number of training samples is not a multiple of the "
+ "batch-size, the last batch will be smaller than the first, unless "
+ "--drop-incomplete--batch is set, in which case this batch is not used.",
+ required=True,
+ show_default=True,
+ default=2,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--drop-incomplete-batch/--no-drop-incomplete-batch",
+ "-D",
+ help="If set, then may drop the last batch in an epoch, in case it is "
+ "incomplete. If you set this option, you should also consider "
+ "increasing the total number of epochs of training, as the total number "
+ "of training steps may be reduced",
+ required=True,
+ show_default=True,
+ default=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--epochs",
+ "-e",
+ help="Number of epochs (complete training set passes) to train for",
+ show_default=True,
+ required=True,
+ default=1000,
+ type=click.IntRange(min=1),
+ cls=ResourceOption,
+)
+@click.option(
+ "--checkpoint-period",
+ "-p",
+ help="Number of epochs after which a checkpoint is saved. "
+ "A value of zero will disable check-pointing. If checkpointing is "
+ "enabled and training stops, it is automatically resumed from the "
+ "last saved checkpoint if training is restarted with the same "
+ "configuration.",
+ show_default=True,
+ required=True,
+ default=0,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@click.option(
+ "--device",
+ "-d",
+ help='A string indicating the device to use (e.g. "cpu" or "cuda:0")',
+ show_default=True,
+ required=True,
+ default="cpu",
+ cls=ResourceOption,
+)
+@click.option(
+ "--seed",
+ "-s",
+ help="Seed to use for the random number generator",
+ show_default=True,
+ required=False,
+ default=42,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@click.option(
+ "--ssl/--no-ssl",
+ help="Switch ON/OFF semi-supervised training mode",
+ show_default=True,
+ required=True,
+ default=False,
+ cls=ResourceOption,
+)
+@click.option(
+ "--rampup",
+ "-r",
+ help="Ramp-up length in epochs (for SSL training only)",
+ show_default=True,
+ required=True,
+ default=900,
+ type=click.IntRange(min=0),
+ cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+def train(
+ model,
+ optimizer,
+ scheduler,
+ output_folder,
+ epochs,
+ pretrained_backbone,
+ batch_size,
+ drop_incomplete_batch,
+ criterion,
+ dataset,
+ checkpoint_period,
+ device,
+ seed,
+ ssl,
+ rampup,
+ verbose,
+ **kwargs,
+):
+ """Trains an FCN to perform binary segmentation
+
+ Training is performed for a configurable number of epochs, and generates at
+ least a final_model.pth. It may also generate a number of intermediate
+ checkpoints. Checkpoints are model files (.pth files) that are stored
+ during the training and useful to resume the procedure in case it stops
+ abruptly.
+ """
+
+ torch.manual_seed(seed)
+
+ use_dataset = dataset
+ if isinstance(dataset, dict):
+ if "__train__" in dataset:
+ logger.info("Found (dedicated) '__train__' set for training")
+ use_dataset = dataset["__train__"]
+ else:
+ use_dataset = dataset["train"]
+
+ # PyTorch dataloader
+ data_loader = DataLoader(
+ dataset=use_dataset,
+ batch_size=batch_size,
+ shuffle=True,
+ drop_last=drop_incomplete_batch,
+ pin_memory=torch.cuda.is_available(),
+ )
+
+ # Checkpointer
+ checkpointer = DetectronCheckpointer(
+ model, optimizer, scheduler, save_dir=output_folder, save_to_disk=True
+ )
+
+ arguments = {}
+ arguments["epoch"] = 0
+ extra_checkpoint_data = checkpointer.load(pretrained_backbone)
+ arguments.update(extra_checkpoint_data)
+ arguments["max_epoch"] = epochs
+
+ logger.info("Training for {} epochs".format(arguments["max_epoch"]))
+ logger.info("Continuing from epoch {}".format(arguments["epoch"]))
+
+ if not ssl:
+ from ..engine.trainer import run
+ run(
+ model,
+ data_loader,
+ optimizer,
+ criterion,
+ scheduler,
+ checkpointer,
+ checkpoint_period,
+ device,
+ arguments,
+ output_folder,
+ )
+
+ else:
+ from ..engine.ssltrainer import run
+ run(
+ model,
+ data_loader,
+ optimizer,
+ criterion,
+ scheduler,
+ checkpointer,
+ checkpoint_period,
+ device,
+ arguments,
+ output_folder,
+ rampup,
+ )
diff --git a/bob/ip/binseg/test/__init__.py b/bob/ip/binseg/test/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e4b13525b1e78beb2b5b4e9a141b3bb5fa2f4f8c 100644
--- a/bob/ip/binseg/test/__init__.py
+++ b/bob/ip/binseg/test/__init__.py
@@ -1,3 +1,76 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Unit tests"""
+
+import tempfile
+import logging
+
+logger = logging.getLogger(__name__)
+
+TESTDB_TMPDIR = None
+_URL = (
+ "http://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/_testdb.zip"
+)
+_RCKEY = "bob.ip.binseg.stare.datadir"
+
+
+def teardown_package():
+ global TESTDB_TMPDIR
+ if TESTDB_TMPDIR is not None:
+ logger.info(f"Removing temporary directory {TESTDB_TMPDIR.name}...")
+ TESTDB_TMPDIR.cleanup()
+
+
+def _mock_test_skipper(name):
+ """
+ Dummary decorator that does nothing
+ """
+ import functools
+
+ def wrapped_function(test):
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ return test(*args, **kwargs)
+
+ return wrapper
+
+ return wrapped_function
+
+
+def mock_dataset():
+ global TESTDB_TMPDIR
+ from bob.extension import rc
+
+ if (TESTDB_TMPDIR is not None) or (_RCKEY in rc):
+ logger.info("Test database already set up - not downloading")
+ else:
+ logger.info("Test database not available, downloading...")
+ import zipfile
+ import urllib.request
+
+ # Download the file from `url` and save it locally under `file_name`:
+ with urllib.request.urlopen(_URL) as r, tempfile.TemporaryFile() as f:
+ f.write(r.read())
+ f.flush()
+ f.seek(0)
+ TESTDB_TMPDIR = tempfile.TemporaryDirectory(prefix=__name__ + "-")
+ print(f"Creating test database at {TESTDB_TMPDIR.name}...")
+ logger.info(f"Creating test database at {TESTDB_TMPDIR.name}...")
+ with zipfile.ZipFile(f) as zf:
+ zf.extractall(TESTDB_TMPDIR.name)
+
+ from ..data import stare
+
+ if TESTDB_TMPDIR is None:
+ # if the user has the STARE directory ready, then we do a normal return
+ from .utils import rc_variable_set
+
+ return rc["bob.ip.binseg.stare.datadir"], stare.dataset, rc_variable_set
+
+ # else, we do a "mock" return
+ return (
+ TESTDB_TMPDIR.name,
+ stare._make_dataset(TESTDB_TMPDIR.name),
+ _mock_test_skipper,
+ )
diff --git a/bob/ip/binseg/test/data/img-16bit.png b/bob/ip/binseg/test/data/img-16bit.png
new file mode 100644
index 0000000000000000000000000000000000000000..1bd5c6ad984e9200e0611d492149126b628fd74d
Binary files /dev/null and b/bob/ip/binseg/test/data/img-16bit.png differ
diff --git a/bob/ip/binseg/test/data/iris-test.csv b/bob/ip/binseg/test/data/iris-test.csv
new file mode 100644
index 0000000000000000000000000000000000000000..27d1b05a7aa70667844b74778504f3b51c624884
--- /dev/null
+++ b/bob/ip/binseg/test/data/iris-test.csv
@@ -0,0 +1,75 @@
+5,3,1.6,0.2,Iris-setosa
+5,3.4,1.6,0.4,Iris-setosa
+5.2,3.5,1.5,0.2,Iris-setosa
+5.2,3.4,1.4,0.2,Iris-setosa
+4.7,3.2,1.6,0.2,Iris-setosa
+4.8,3.1,1.6,0.2,Iris-setosa
+5.4,3.4,1.5,0.4,Iris-setosa
+5.2,4.1,1.5,0.1,Iris-setosa
+5.5,4.2,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5,3.2,1.2,0.2,Iris-setosa
+5.5,3.5,1.3,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+4.4,3,1.3,0.2,Iris-setosa
+5.1,3.4,1.5,0.2,Iris-setosa
+5,3.5,1.3,0.3,Iris-setosa
+4.5,2.3,1.3,0.3,Iris-setosa
+4.4,3.2,1.3,0.2,Iris-setosa
+5,3.5,1.6,0.6,Iris-setosa
+5.1,3.8,1.9,0.4,Iris-setosa
+4.8,3,1.4,0.3,Iris-setosa
+5.1,3.8,1.6,0.2,Iris-setosa
+4.6,3.2,1.4,0.2,Iris-setosa
+5.3,3.7,1.5,0.2,Iris-setosa
+5,3.3,1.4,0.2,Iris-setosa
+6.6,3,4.4,1.4,Iris-versicolor
+6.8,2.8,4.8,1.4,Iris-versicolor
+6.7,3,5,1.7,Iris-versicolor
+6,2.9,4.5,1.5,Iris-versicolor
+5.7,2.6,3.5,1,Iris-versicolor
+5.5,2.4,3.8,1.1,Iris-versicolor
+5.5,2.4,3.7,1,Iris-versicolor
+5.8,2.7,3.9,1.2,Iris-versicolor
+6,2.7,5.1,1.6,Iris-versicolor
+5.4,3,4.5,1.5,Iris-versicolor
+6,3.4,4.5,1.6,Iris-versicolor
+6.7,3.1,4.7,1.5,Iris-versicolor
+6.3,2.3,4.4,1.3,Iris-versicolor
+5.6,3,4.1,1.3,Iris-versicolor
+5.5,2.5,4,1.3,Iris-versicolor
+5.5,2.6,4.4,1.2,Iris-versicolor
+6.1,3,4.6,1.4,Iris-versicolor
+5.8,2.6,4,1.2,Iris-versicolor
+5,2.3,3.3,1,Iris-versicolor
+5.6,2.7,4.2,1.3,Iris-versicolor
+5.7,3,4.2,1.2,Iris-versicolor
+5.7,2.9,4.2,1.3,Iris-versicolor
+6.2,2.9,4.3,1.3,Iris-versicolor
+5.1,2.5,3,1.1,Iris-versicolor
+5.7,2.8,4.1,1.3,Iris-versicolor
+7.2,3.2,6,1.8,Iris-virginica
+6.2,2.8,4.8,1.8,Iris-virginica
+6.1,3,4.9,1.8,Iris-virginica
+6.4,2.8,5.6,2.1,Iris-virginica
+7.2,3,5.8,1.6,Iris-virginica
+7.4,2.8,6.1,1.9,Iris-virginica
+7.9,3.8,6.4,2,Iris-virginica
+6.4,2.8,5.6,2.2,Iris-virginica
+6.3,2.8,5.1,1.5,Iris-virginica
+6.1,2.6,5.6,1.4,Iris-virginica
+7.7,3,6.1,2.3,Iris-virginica
+6.3,3.4,5.6,2.4,Iris-virginica
+6.4,3.1,5.5,1.8,Iris-virginica
+6,3,4.8,1.8,Iris-virginica
+6.9,3.1,5.4,2.1,Iris-virginica
+6.7,3.1,5.6,2.4,Iris-virginica
+6.9,3.1,5.1,2.3,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+6.8,3.2,5.9,2.3,Iris-virginica
+6.7,3.3,5.7,2.5,Iris-virginica
+6.7,3,5.2,2.3,Iris-virginica
+6.3,2.5,5,1.9,Iris-virginica
+6.5,3,5.2,2,Iris-virginica
+6.2,3.4,5.4,2.3,Iris-virginica
+5.9,3,5.1,1.8,Iris-virginica
diff --git a/bob/ip/binseg/test/data/iris-train.csv b/bob/ip/binseg/test/data/iris-train.csv
new file mode 100644
index 0000000000000000000000000000000000000000..82d5b134803975463f070aebe6847e7c742749d2
--- /dev/null
+++ b/bob/ip/binseg/test/data/iris-train.csv
@@ -0,0 +1,75 @@
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.4,3.7,1.5,0.2,Iris-setosa
+4.8,3.4,1.6,0.2,Iris-setosa
+4.8,3,1.4,0.1,Iris-setosa
+4.3,3,1.1,0.1,Iris-setosa
+5.8,4,1.2,0.2,Iris-setosa
+5.7,4.4,1.5,0.4,Iris-setosa
+5.4,3.9,1.3,0.4,Iris-setosa
+5.1,3.5,1.4,0.3,Iris-setosa
+5.7,3.8,1.7,0.3,Iris-setosa
+5.1,3.8,1.5,0.3,Iris-setosa
+5.4,3.4,1.7,0.2,Iris-setosa
+5.1,3.7,1.5,0.4,Iris-setosa
+4.6,3.6,1,0.2,Iris-setosa
+5.1,3.3,1.7,0.5,Iris-setosa
+4.8,3.4,1.9,0.2,Iris-setosa
+7,3.2,4.7,1.4,Iris-versicolor
+6.4,3.2,4.5,1.5,Iris-versicolor
+6.9,3.1,4.9,1.5,Iris-versicolor
+5.5,2.3,4,1.3,Iris-versicolor
+6.5,2.8,4.6,1.5,Iris-versicolor
+5.7,2.8,4.5,1.3,Iris-versicolor
+6.3,3.3,4.7,1.6,Iris-versicolor
+4.9,2.4,3.3,1,Iris-versicolor
+6.6,2.9,4.6,1.3,Iris-versicolor
+5.2,2.7,3.9,1.4,Iris-versicolor
+5,2,3.5,1,Iris-versicolor
+5.9,3,4.2,1.5,Iris-versicolor
+6,2.2,4,1,Iris-versicolor
+6.1,2.9,4.7,1.4,Iris-versicolor
+5.6,2.9,3.6,1.3,Iris-versicolor
+6.7,3.1,4.4,1.4,Iris-versicolor
+5.6,3,4.5,1.5,Iris-versicolor
+5.8,2.7,4.1,1,Iris-versicolor
+6.2,2.2,4.5,1.5,Iris-versicolor
+5.6,2.5,3.9,1.1,Iris-versicolor
+5.9,3.2,4.8,1.8,Iris-versicolor
+6.1,2.8,4,1.3,Iris-versicolor
+6.3,2.5,4.9,1.5,Iris-versicolor
+6.1,2.8,4.7,1.2,Iris-versicolor
+6.4,2.9,4.3,1.3,Iris-versicolor
+6.3,3.3,6,2.5,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+7.1,3,5.9,2.1,Iris-virginica
+6.3,2.9,5.6,1.8,Iris-virginica
+6.5,3,5.8,2.2,Iris-virginica
+7.6,3,6.6,2.1,Iris-virginica
+4.9,2.5,4.5,1.7,Iris-virginica
+7.3,2.9,6.3,1.8,Iris-virginica
+6.7,2.5,5.8,1.8,Iris-virginica
+7.2,3.6,6.1,2.5,Iris-virginica
+6.5,3.2,5.1,2,Iris-virginica
+6.4,2.7,5.3,1.9,Iris-virginica
+6.8,3,5.5,2.1,Iris-virginica
+5.7,2.5,5,2,Iris-virginica
+5.8,2.8,5.1,2.4,Iris-virginica
+6.4,3.2,5.3,2.3,Iris-virginica
+6.5,3,5.5,1.8,Iris-virginica
+7.7,3.8,6.7,2.2,Iris-virginica
+7.7,2.6,6.9,2.3,Iris-virginica
+6,2.2,5,1.5,Iris-virginica
+6.9,3.2,5.7,2.3,Iris-virginica
+5.6,2.8,4.9,2,Iris-virginica
+7.7,2.8,6.7,2,Iris-virginica
+6.3,2.7,4.9,1.8,Iris-virginica
+6.7,3.3,5.7,2.1,Iris-virginica
diff --git a/bob/ip/binseg/test/data/iris.json b/bob/ip/binseg/test/data/iris.json
new file mode 100644
index 0000000000000000000000000000000000000000..1777efc361562880096b9a21b2011218c152ecac
--- /dev/null
+++ b/bob/ip/binseg/test/data/iris.json
@@ -0,0 +1,156 @@
+{
+ "train": [
+ [5.1,3.5,1.4,0.2,"Iris-setosa"],
+ [4.9,3,1.4,0.2,"Iris-setosa"],
+ [4.7,3.2,1.3,0.2,"Iris-setosa"],
+ [4.6,3.1,1.5,0.2,"Iris-setosa"],
+ [5,3.6,1.4,0.2,"Iris-setosa"],
+ [5.4,3.9,1.7,0.4,"Iris-setosa"],
+ [4.6,3.4,1.4,0.3,"Iris-setosa"],
+ [5,3.4,1.5,0.2,"Iris-setosa"],
+ [4.4,2.9,1.4,0.2,"Iris-setosa"],
+ [4.9,3.1,1.5,0.1,"Iris-setosa"],
+ [5.4,3.7,1.5,0.2,"Iris-setosa"],
+ [4.8,3.4,1.6,0.2,"Iris-setosa"],
+ [4.8,3,1.4,0.1,"Iris-setosa"],
+ [4.3,3,1.1,0.1,"Iris-setosa"],
+ [5.8,4,1.2,0.2,"Iris-setosa"],
+ [5.7,4.4,1.5,0.4,"Iris-setosa"],
+ [5.4,3.9,1.3,0.4,"Iris-setosa"],
+ [5.1,3.5,1.4,0.3,"Iris-setosa"],
+ [5.7,3.8,1.7,0.3,"Iris-setosa"],
+ [5.1,3.8,1.5,0.3,"Iris-setosa"],
+ [5.4,3.4,1.7,0.2,"Iris-setosa"],
+ [5.1,3.7,1.5,0.4,"Iris-setosa"],
+ [4.6,3.6,1,0.2,"Iris-setosa"],
+ [5.1,3.3,1.7,0.5,"Iris-setosa"],
+ [4.8,3.4,1.9,0.2,"Iris-setosa"],
+ [7,3.2,4.7,1.4,"Iris-versicolor"],
+ [6.4,3.2,4.5,1.5,"Iris-versicolor"],
+ [6.9,3.1,4.9,1.5,"Iris-versicolor"],
+ [5.5,2.3,4,1.3,"Iris-versicolor"],
+ [6.5,2.8,4.6,1.5,"Iris-versicolor"],
+ [5.7,2.8,4.5,1.3,"Iris-versicolor"],
+ [6.3,3.3,4.7,1.6,"Iris-versicolor"],
+ [4.9,2.4,3.3,1,"Iris-versicolor"],
+ [6.6,2.9,4.6,1.3,"Iris-versicolor"],
+ [5.2,2.7,3.9,1.4,"Iris-versicolor"],
+ [5,2,3.5,1,"Iris-versicolor"],
+ [5.9,3,4.2,1.5,"Iris-versicolor"],
+ [6,2.2,4,1,"Iris-versicolor"],
+ [6.1,2.9,4.7,1.4,"Iris-versicolor"],
+ [5.6,2.9,3.6,1.3,"Iris-versicolor"],
+ [6.7,3.1,4.4,1.4,"Iris-versicolor"],
+ [5.6,3,4.5,1.5,"Iris-versicolor"],
+ [5.8,2.7,4.1,1,"Iris-versicolor"],
+ [6.2,2.2,4.5,1.5,"Iris-versicolor"],
+ [5.6,2.5,3.9,1.1,"Iris-versicolor"],
+ [5.9,3.2,4.8,1.8,"Iris-versicolor"],
+ [6.1,2.8,4,1.3,"Iris-versicolor"],
+ [6.3,2.5,4.9,1.5,"Iris-versicolor"],
+ [6.1,2.8,4.7,1.2,"Iris-versicolor"],
+ [6.4,2.9,4.3,1.3,"Iris-versicolor"],
+ [6.3,3.3,6,2.5,"Iris-virginica"],
+ [5.8,2.7,5.1,1.9,"Iris-virginica"],
+ [7.1,3,5.9,2.1,"Iris-virginica"],
+ [6.3,2.9,5.6,1.8,"Iris-virginica"],
+ [6.5,3,5.8,2.2,"Iris-virginica"],
+ [7.6,3,6.6,2.1,"Iris-virginica"],
+ [4.9,2.5,4.5,1.7,"Iris-virginica"],
+ [7.3,2.9,6.3,1.8,"Iris-virginica"],
+ [6.7,2.5,5.8,1.8,"Iris-virginica"],
+ [7.2,3.6,6.1,2.5,"Iris-virginica"],
+ [6.5,3.2,5.1,2,"Iris-virginica"],
+ [6.4,2.7,5.3,1.9,"Iris-virginica"],
+ [6.8,3,5.5,2.1,"Iris-virginica"],
+ [5.7,2.5,5,2,"Iris-virginica"],
+ [5.8,2.8,5.1,2.4,"Iris-virginica"],
+ [6.4,3.2,5.3,2.3,"Iris-virginica"],
+ [6.5,3,5.5,1.8,"Iris-virginica"],
+ [7.7,3.8,6.7,2.2,"Iris-virginica"],
+ [7.7,2.6,6.9,2.3,"Iris-virginica"],
+ [6,2.2,5,1.5,"Iris-virginica"],
+ [6.9,3.2,5.7,2.3,"Iris-virginica"],
+ [5.6,2.8,4.9,2,"Iris-virginica"],
+ [7.7,2.8,6.7,2,"Iris-virginica"],
+ [6.3,2.7,4.9,1.8,"Iris-virginica"],
+ [6.7,3.3,5.7,2.1,"Iris-virginica"]
+ ],
+ "test": [
+ [5,3,1.6,0.2,"Iris-setosa"],
+ [5,3.4,1.6,0.4,"Iris-setosa"],
+ [5.2,3.5,1.5,0.2,"Iris-setosa"],
+ [5.2,3.4,1.4,0.2,"Iris-setosa"],
+ [4.7,3.2,1.6,0.2,"Iris-setosa"],
+ [4.8,3.1,1.6,0.2,"Iris-setosa"],
+ [5.4,3.4,1.5,0.4,"Iris-setosa"],
+ [5.2,4.1,1.5,0.1,"Iris-setosa"],
+ [5.5,4.2,1.4,0.2,"Iris-setosa"],
+ [4.9,3.1,1.5,0.1,"Iris-setosa"],
+ [5,3.2,1.2,0.2,"Iris-setosa"],
+ [5.5,3.5,1.3,0.2,"Iris-setosa"],
+ [4.9,3.1,1.5,0.1,"Iris-setosa"],
+ [4.4,3,1.3,0.2,"Iris-setosa"],
+ [5.1,3.4,1.5,0.2,"Iris-setosa"],
+ [5,3.5,1.3,0.3,"Iris-setosa"],
+ [4.5,2.3,1.3,0.3,"Iris-setosa"],
+ [4.4,3.2,1.3,0.2,"Iris-setosa"],
+ [5,3.5,1.6,0.6,"Iris-setosa"],
+ [5.1,3.8,1.9,0.4,"Iris-setosa"],
+ [4.8,3,1.4,0.3,"Iris-setosa"],
+ [5.1,3.8,1.6,0.2,"Iris-setosa"],
+ [4.6,3.2,1.4,0.2,"Iris-setosa"],
+ [5.3,3.7,1.5,0.2,"Iris-setosa"],
+ [5,3.3,1.4,0.2,"Iris-setosa"],
+ [6.6,3,4.4,1.4,"Iris-versicolor"],
+ [6.8,2.8,4.8,1.4,"Iris-versicolor"],
+ [6.7,3,5,1.7,"Iris-versicolor"],
+ [6,2.9,4.5,1.5,"Iris-versicolor"],
+ [5.7,2.6,3.5,1,"Iris-versicolor"],
+ [5.5,2.4,3.8,1.1,"Iris-versicolor"],
+ [5.5,2.4,3.7,1,"Iris-versicolor"],
+ [5.8,2.7,3.9,1.2,"Iris-versicolor"],
+ [6,2.7,5.1,1.6,"Iris-versicolor"],
+ [5.4,3,4.5,1.5,"Iris-versicolor"],
+ [6,3.4,4.5,1.6,"Iris-versicolor"],
+ [6.7,3.1,4.7,1.5,"Iris-versicolor"],
+ [6.3,2.3,4.4,1.3,"Iris-versicolor"],
+ [5.6,3,4.1,1.3,"Iris-versicolor"],
+ [5.5,2.5,4,1.3,"Iris-versicolor"],
+ [5.5,2.6,4.4,1.2,"Iris-versicolor"],
+ [6.1,3,4.6,1.4,"Iris-versicolor"],
+ [5.8,2.6,4,1.2,"Iris-versicolor"],
+ [5,2.3,3.3,1,"Iris-versicolor"],
+ [5.6,2.7,4.2,1.3,"Iris-versicolor"],
+ [5.7,3,4.2,1.2,"Iris-versicolor"],
+ [5.7,2.9,4.2,1.3,"Iris-versicolor"],
+ [6.2,2.9,4.3,1.3,"Iris-versicolor"],
+ [5.1,2.5,3,1.1,"Iris-versicolor"],
+ [5.7,2.8,4.1,1.3,"Iris-versicolor"],
+ [7.2,3.2,6,1.8,"Iris-virginica"],
+ [6.2,2.8,4.8,1.8,"Iris-virginica"],
+ [6.1,3,4.9,1.8,"Iris-virginica"],
+ [6.4,2.8,5.6,2.1,"Iris-virginica"],
+ [7.2,3,5.8,1.6,"Iris-virginica"],
+ [7.4,2.8,6.1,1.9,"Iris-virginica"],
+ [7.9,3.8,6.4,2,"Iris-virginica"],
+ [6.4,2.8,5.6,2.2,"Iris-virginica"],
+ [6.3,2.8,5.1,1.5,"Iris-virginica"],
+ [6.1,2.6,5.6,1.4,"Iris-virginica"],
+ [7.7,3,6.1,2.3,"Iris-virginica"],
+ [6.3,3.4,5.6,2.4,"Iris-virginica"],
+ [6.4,3.1,5.5,1.8,"Iris-virginica"],
+ [6,3,4.8,1.8,"Iris-virginica"],
+ [6.9,3.1,5.4,2.1,"Iris-virginica"],
+ [6.7,3.1,5.6,2.4,"Iris-virginica"],
+ [6.9,3.1,5.1,2.3,"Iris-virginica"],
+ [5.8,2.7,5.1,1.9,"Iris-virginica"],
+ [6.8,3.2,5.9,2.3,"Iris-virginica"],
+ [6.7,3.3,5.7,2.5,"Iris-virginica"],
+ [6.7,3,5.2,2.3,"Iris-virginica"],
+ [6.3,2.5,5,1.9,"Iris-virginica"],
+ [6.5,3,5.2,2,"Iris-virginica"],
+ [6.2,3.4,5.4,2.3,"Iris-virginica"],
+ [5.9,3,5.1,1.8,"Iris-virginica"]
+ ]
+}
diff --git a/bob/ip/binseg/test/test_basemetrics.py b/bob/ip/binseg/test/test_basemetrics.py
index bf478ac788d038dd9038ca5d5b5cf7aa1ac7a83a..969894f5e453bfdf6fc86fe07448d8e1c8f7ece2 100644
--- a/bob/ip/binseg/test/test_basemetrics.py
+++ b/bob/ip/binseg/test/test_basemetrics.py
@@ -2,43 +2,47 @@
# -*- coding: utf-8 -*-
import unittest
-import numpy as np
from bob.ip.binseg.utils.metric import base_metrics
import random
+
class Tester(unittest.TestCase):
"""
Unit test for base metrics
"""
+
def setUp(self):
self.tp = random.randint(1, 100)
self.fp = random.randint(1, 100)
self.tn = random.randint(1, 100)
self.fn = random.randint(1, 100)
-
+
def test_precision(self):
precision = base_metrics(self.tp, self.fp, self.tn, self.fn)[0]
- self.assertEqual((self.tp)/(self.tp + self.fp),precision)
+ self.assertEqual((self.tp) / (self.tp + self.fp), precision)
def test_recall(self):
recall = base_metrics(self.tp, self.fp, self.tn, self.fn)[1]
- self.assertEqual((self.tp)/(self.tp + self.fn),recall)
+ self.assertEqual((self.tp) / (self.tp + self.fn), recall)
def test_specificity(self):
specificity = base_metrics(self.tp, self.fp, self.tn, self.fn)[2]
- self.assertEqual((self.tn)/(self.tn + self.fp),specificity)
-
+ self.assertEqual((self.tn) / (self.tn + self.fp), specificity)
+
def test_accuracy(self):
accuracy = base_metrics(self.tp, self.fp, self.tn, self.fn)[3]
- self.assertEqual((self.tp + self.tn)/(self.tp + self.tn + self.fp + self.fn), accuracy)
+ self.assertEqual(
+ (self.tp + self.tn) / (self.tp + self.tn + self.fp + self.fn), accuracy
+ )
def test_jaccard(self):
jaccard = base_metrics(self.tp, self.fp, self.tn, self.fn)[4]
- self.assertEqual(self.tp / (self.tp+self.fp+self.fn), jaccard)
+ self.assertEqual(self.tp / (self.tp + self.fp + self.fn), jaccard)
def test_f1(self):
f1 = base_metrics(self.tp, self.fp, self.tn, self.fn)[5]
- self.assertEqual((2.0 * self.tp ) / (2.0 * self.tp + self.fp + self.fn ),f1)
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ self.assertEqual((2.0 * self.tp) / (2.0 * self.tp + self.fp + self.fn), f1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/bob/ip/binseg/test/test_batchmetrics.py b/bob/ip/binseg/test/test_batchmetrics.py
index 4988cab6ea8a7ffbc907f2e580ebe46b48ad9611..09ffe250a805a00718d2cb4687ea2bbe5e49daf4 100644
--- a/bob/ip/binseg/test/test_batchmetrics.py
+++ b/bob/ip/binseg/test/test_batchmetrics.py
@@ -2,38 +2,45 @@
# -*- coding: utf-8 -*-
import unittest
-import numpy as np
-from bob.ip.binseg.engine.inferencer import batch_metrics
import random
-import shutil, tempfile
-import logging
+import shutil
+
import torch
+import pandas
+import numpy
+
+from ..engine.evaluator import _sample_metrics
+
+import logging
+logger = logging.getLogger(__name__)
+
class Tester(unittest.TestCase):
"""
Unit test for batch metrics
"""
+
def setUp(self):
self.tp = random.randint(1, 100)
self.fp = random.randint(1, 100)
self.tn = random.randint(1, 100)
self.fn = random.randint(1, 100)
- self.predictions = torch.rand(size=(2,1,420,420))
- self.ground_truths = torch.randint(low=0, high=2, size=(2,1,420,420))
- self.names = ['Bob','Tim']
- self.output_folder = tempfile.mkdtemp()
- self.logger = logging.getLogger(__name__)
-
- def tearDown(self):
- # Remove the temporary folder after the test
- shutil.rmtree(self.output_folder)
-
+ self.predictions = torch.rand(size=(2, 1, 420, 420))
+ self.ground_truths = torch.randint(low=0, high=2, size=(2, 1, 420, 420))
+ self.names = ["Bob", "Tim"]
+
def test_batch_metrics(self):
- bm = batch_metrics(self.predictions, self.ground_truths, self.names, self.output_folder, self.logger)
- self.assertEqual(len(bm),2*100)
- for metric in bm:
- # check whether f1 score agree
- self.assertAlmostEqual(metric[-1],2*(metric[-6]*metric[-5])/(metric[-6]+metric[-5]))
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ dfs = []
+ for pred, gt in zip(self.predictions, self.ground_truths):
+ dfs.append(_sample_metrics(pred, gt, 100))
+ bm = pandas.concat(dfs)
+
+ self.assertEqual(len(bm), 2 * 100)
+ # check whether f1 score agree
+ calculated = bm.f1_score.to_numpy()
+ ours = (2*(bm.precision*bm.recall)/(bm.precision+bm.recall)).to_numpy()
+ assert numpy.isclose(calculated, ours).all()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/bob/ip/binseg/test/test_chasedb1.py b/bob/ip/binseg/test/test_chasedb1.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5a980c14f2992307a53e69924cb6b7c314fc538
--- /dev/null
+++ b/bob/ip/binseg/test/test_chasedb1.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for CHASE-DB1"""
+
+import os
+
+import numpy
+import nose.tools
+
+from ..data.chasedb1 import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ subset = dataset.subsets("first-annotator")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 8)
+ for s in subset["train"]:
+ assert s.key.startswith("Image_")
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 20)
+ for s in subset["test"]:
+ assert s.key.startswith("Image_")
+
+ subset = dataset.subsets("second-annotator")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 8)
+ for s in subset["train"]:
+ assert s.key.startswith("Image_")
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 20)
+ for s in subset["test"]:
+ assert s.key.startswith("Image_")
+
+
+@rc_variable_set('bob.ip.binseg.chasedb1.datadir')
+def test_loading():
+
+ image_size = (999, 960)
+
+ def _check_sample(s, bw_threshold_label):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 2)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("first-annotator")
+ proportions = [_check_sample(s, 0.08) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.10) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("second-annotator")
+ proportions = [_check_sample(s, 0.09) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.09) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+
+@rc_variable_set('bob.ip.binseg.chasedb1.datadir')
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_checkpointer.py b/bob/ip/binseg/test/test_checkpointer.py
index 45a181b39a298fb0a7340ad745113543c4048b2f..52bd9ac0fafefd1d7d250bfcd87c4d3e6703ad15 100644
--- a/bob/ip/binseg/test/test_checkpointer.py
+++ b/bob/ip/binseg/test/test_checkpointer.py
@@ -1,8 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/engine/trainer.py
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from collections import OrderedDict
from tempfile import TemporaryDirectory
import unittest
@@ -41,9 +39,7 @@ class TestCheckpointer(unittest.TestCase):
trained_model = self.create_model()
fresh_model = self.create_model()
with TemporaryDirectory() as f:
- checkpointer = Checkpointer(
- trained_model, save_dir=f, save_to_disk=True
- )
+ checkpointer = Checkpointer(trained_model, save_dir=f, save_to_disk=True)
checkpointer.save("checkpoint_file")
# in the same folder
@@ -51,7 +47,7 @@ class TestCheckpointer(unittest.TestCase):
self.assertTrue(fresh_checkpointer.has_checkpoint())
self.assertEqual(
fresh_checkpointer.get_checkpoint_file(),
- os.path.join(f, "checkpoint_file.pth"),
+ "checkpoint_file.pth",
)
_ = fresh_checkpointer.load()
@@ -68,9 +64,7 @@ class TestCheckpointer(unittest.TestCase):
trained_model = self.create_model()
fresh_model = self.create_model()
with TemporaryDirectory() as f:
- checkpointer = Checkpointer(
- trained_model, save_dir=f, save_to_disk=True
- )
+ checkpointer = Checkpointer(trained_model, save_dir=f, save_to_disk=True)
checkpointer.save("checkpoint_file")
# on different folders
@@ -90,4 +84,4 @@ class TestCheckpointer(unittest.TestCase):
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/bob/ip/binseg/test/test_cli.py b/bob/ip/binseg/test/test_cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b0d20af4fb3fe312afb2c95ed7acd577d737b40
--- /dev/null
+++ b/bob/ip/binseg/test/test_cli.py
@@ -0,0 +1,573 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Tests for our CLI applications"""
+
+import os
+import re
+import fnmatch
+import tempfile
+import contextlib
+
+import nose.tools
+
+from click.testing import CliRunner
+
+from . import mock_dataset
+
+stare_datadir, stare_dataset, rc_variable_set = mock_dataset()
+
+
+@contextlib.contextmanager
+def stdout_logging():
+
+ ## copy logging messages to std out
+ import sys
+ import logging
+ import io
+
+ buf = io.StringIO()
+ ch = logging.StreamHandler(buf)
+ ch.setFormatter(logging.Formatter("%(message)s"))
+ ch.setLevel(logging.INFO)
+ logger = logging.getLogger("bob")
+ logger.addHandler(ch)
+ yield buf
+ logger.removeHandler(ch)
+
+
+def _assert_exit_0(result):
+
+ assert (
+ result.exit_code == 0
+ ), f"Exit code {result.exit_code} != 0 -- Output:\n{result.output}"
+
+
+def _check_help(entry_point):
+
+ runner = CliRunner()
+ result = runner.invoke(entry_point, ["--help"])
+ _assert_exit_0(result)
+ assert result.output.startswith("Usage:")
+
+
+def test_main_help():
+ from ..script.binseg import binseg
+
+ _check_help(binseg)
+
+
+def test_experiment_help():
+ from ..script.experiment import experiment
+
+ _check_help(experiment)
+
+
+def _str_counter(substr, s):
+ return sum(1 for _ in re.finditer(substr, s, re.MULTILINE))
+
+
+def _check_experiment_stare(overlay):
+
+ from ..script.experiment import experiment
+
+ runner = CliRunner()
+ with runner.isolated_filesystem(), stdout_logging() as buf, tempfile.NamedTemporaryFile(
+ mode="wt"
+ ) as config:
+
+ # re-write STARE dataset configuration for test
+ config.write("from bob.ip.binseg.data.stare import _make_dataset\n")
+ config.write(f"_raw = _make_dataset('{stare_datadir}')\n")
+ config.write(
+ "from bob.ip.binseg.configs.datasets.stare import _maker\n"
+ )
+ config.write("dataset = _maker('ah', _raw)\n")
+ config.write("second_annotator = _maker('vk', _raw)\n")
+ config.flush()
+
+ output_folder = "results"
+ options = [
+ "m2unet",
+ config.name,
+ "-vv",
+ "--epochs=1",
+ "--batch-size=1",
+ f"--output-folder={output_folder}",
+ ]
+ if overlay:
+ options += ["--overlayed"]
+ result = runner.invoke(experiment, options)
+ _assert_exit_0(result)
+
+ # check command-line
+ assert os.path.exists(os.path.join(output_folder, "command.sh"))
+
+ # check model was saved
+ train_folder = os.path.join(output_folder, "model")
+ assert os.path.exists(os.path.join(train_folder, "model_final.pth"))
+ assert os.path.exists(os.path.join(train_folder, "last_checkpoint"))
+ assert os.path.exists(os.path.join(train_folder, "constants.csv"))
+ assert os.path.exists(os.path.join(train_folder, "trainlog.csv"))
+ assert os.path.exists(os.path.join(train_folder, "model_summary.txt"))
+
+ # check predictions are there
+ predict_folder = os.path.join(output_folder, "predictions")
+ basedir = os.path.join(predict_folder, "stare-images")
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 20)
+
+ overlay_folder = os.path.join(output_folder, "overlayed", "predictions")
+ basedir = os.path.join(overlay_folder, "stare-images")
+ if overlay:
+ # check overlayed images are there (since we requested them)
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
+
+ # check evaluation outputs
+ eval_folder = os.path.join(output_folder, "analysis")
+ assert os.path.exists(os.path.join(eval_folder, "train.csv"))
+ assert os.path.exists(os.path.join(eval_folder, "test.csv"))
+ assert os.path.exists(
+ os.path.join(eval_folder, "second-annotator", "train.csv")
+ )
+ assert os.path.exists(
+ os.path.join(eval_folder, "second-annotator" , "test.csv")
+ )
+
+ overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
+ basedir = os.path.join(overlay_folder, "stare-images")
+ if overlay:
+ # check overlayed images are there (since we requested them)
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
+
+ # check overlayed images from first-to-second annotator comparisons
+ # are there (since we requested them)
+ overlay_folder = os.path.join(output_folder, "overlayed", "analysis",
+ "second-annotator")
+ basedir = os.path.join(overlay_folder, "stare-images")
+ if overlay:
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
+
+ # check outcomes of the comparison phase
+ assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))
+ assert os.path.exists(os.path.join(output_folder, "comparison.rst"))
+
+ keywords = {
+ r"^Started training$": 1,
+ r"^Found \(dedicated\) '__train__' set for training$": 1,
+ r"^Continuing from epoch 0$": 1,
+ r"^Saving model summary at.*$": 1,
+ r"^Model has.*$": 1,
+ r"^Saving checkpoint": 1,
+ r"^Ended training$": 1,
+ r"^Started prediction$": 1,
+ r"^Loading checkpoint from": 2,
+ r"^Ended prediction$": 1,
+ r"^Started evaluation$": 1,
+ r"^Maximum F1-score of.*\(chosen \*a posteriori\*\)$": 3,
+ r"^F1-score of.*\(chosen \*a priori\*\)$": 2,
+ r"^F1-score of.*\(second annotator; threshold=0.5\)$": 2,
+ r"^Ended evaluation$": 1,
+ r"^Started comparison$": 1,
+ r"^Loading metrics from": 4,
+ r"^Creating and saving plot at": 1,
+ r"^Tabulating performance summary...": 1,
+ r"^Saving table at": 1,
+ r"^Ended comparison.*$": 1,
+ }
+ buf.seek(0)
+ logging_output = buf.read()
+ for k, v in keywords.items():
+ # if _str_counter(k, logging_output) != v:
+ # print(f"Count for string '{k}' appeared " \
+ # f"({_str_counter(k, result.output)}) " \
+ # f"instead of the expected {v}")
+ assert _str_counter(k, logging_output) == v, (
+ f"Count for string '{k}' appeared "
+ f"({_str_counter(k, logging_output)}) "
+ f"instead of the expected {v}"
+ )
+
+
+@rc_variable_set("bob.ip.binseg.stare.datadir")
+def test_experiment_stare_with_overlay():
+ _check_experiment_stare(overlay=True)
+
+
+@rc_variable_set("bob.ip.binseg.stare.datadir")
+def test_experiment_stare_without_overlay():
+ _check_experiment_stare(overlay=False)
+
+
+def _check_train(runner):
+
+ from ..script.train import train
+
+ with tempfile.NamedTemporaryFile(
+ mode="wt"
+ ) as config, stdout_logging() as buf:
+
+ # single training set configuration
+ config.write("from bob.ip.binseg.data.stare import _make_dataset\n")
+ config.write(f"_raw = _make_dataset('{stare_datadir}')\n")
+ config.write(
+ "from bob.ip.binseg.configs.datasets.stare import _maker\n"
+ )
+ config.write("dataset = _maker('ah', _raw)['train']\n")
+ config.flush()
+
+ output_folder = "results"
+ result = runner.invoke(
+ train,
+ [
+ "m2unet",
+ config.name,
+ "-vv",
+ "--epochs=1",
+ "--batch-size=1",
+ f"--output-folder={output_folder}",
+ ],
+ )
+ _assert_exit_0(result)
+
+ assert os.path.exists(os.path.join(output_folder, "model_final.pth"))
+ assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
+ assert os.path.exists(os.path.join(output_folder, "constants.csv"))
+ assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
+
+ keywords = {
+ r"^Continuing from epoch 0$": 1,
+ r"^Saving model summary at.*$": 1,
+ r"^Model has.*$": 1,
+ rf"^Saving checkpoint to {output_folder}/model_final.pth$": 1,
+ r"^Total training time:": 1,
+ }
+ buf.seek(0)
+ logging_output = buf.read()
+
+ for k, v in keywords.items():
+ # if _str_counter(k, logging_output) != v:
+ # print(f"Count for string '{k}' appeared " \
+ # f"({_str_counter(k, result.output)}) " \
+ # f"instead of the expected {v}")
+ assert _str_counter(k, logging_output) == v, (
+ f"Count for string '{k}' appeared "
+ f"({_str_counter(k, logging_output)}) "
+ f"instead of the expected {v}:\nOutput:\n{logging_output}"
+ )
+
+
+def _check_predict(runner):
+
+ from ..script.predict import predict
+
+ with tempfile.NamedTemporaryFile(
+ mode="wt"
+ ) as config, stdout_logging() as buf:
+
+ # single training set configuration
+ config.write("from bob.ip.binseg.data.stare import _make_dataset\n")
+ config.write(f"_raw = _make_dataset('{stare_datadir}')\n")
+ config.write(
+ "from bob.ip.binseg.configs.datasets.stare import _maker\n"
+ )
+ config.write("dataset = _maker('ah', _raw)['test']\n")
+ config.flush()
+
+ output_folder = "predictions"
+ overlay_folder = os.path.join("overlayed", "predictions")
+ result = runner.invoke(
+ predict,
+ [
+ "m2unet",
+ config.name,
+ "-vv",
+ "--batch-size=1",
+ "--weight=results/model_final.pth",
+ f"--output-folder={output_folder}",
+ f"--overlayed={overlay_folder}",
+ ],
+ )
+ _assert_exit_0(result)
+
+ # check predictions are there
+ basedir = os.path.join(output_folder, "stare-images")
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 10)
+
+ # check overlayed images are there (since we requested them)
+ basedir = os.path.join(overlay_folder, "stare-images")
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
+
+ keywords = {
+ r"^Loading checkpoint from.*$": 1,
+ r"^Total time:.*$": 1,
+ }
+ buf.seek(0)
+ logging_output = buf.read()
+
+ for k, v in keywords.items():
+ # if _str_counter(k, logging_output) != v:
+ # print(f"Count for string '{k}' appeared " \
+ # f"({_str_counter(k, result.output)}) " \
+ # f"instead of the expected {v}")
+ assert _str_counter(k, logging_output) == v, (
+ f"Count for string '{k}' appeared "
+ f"({_str_counter(k, logging_output)}) "
+ f"instead of the expected {v}:\nOutput:\n{logging_output}"
+ )
+
+
+def _check_evaluate(runner):
+
+ from ..script.evaluate import evaluate
+
+ with tempfile.NamedTemporaryFile(
+ mode="wt"
+ ) as config, stdout_logging() as buf:
+
+ # single training set configuration
+ config.write("from bob.ip.binseg.data.stare import _make_dataset\n")
+ config.write(f"_raw = _make_dataset('{stare_datadir}')\n")
+ config.write(
+ "from bob.ip.binseg.configs.datasets.stare import _maker\n"
+ )
+ config.write("dataset = _maker('ah', _raw)['test']\n")
+ config.write("second_annotator = _maker('vk', _raw)['test']\n")
+ config.flush()
+
+ output_folder = "evaluations"
+ overlay_folder = os.path.join("overlayed", "analysis")
+ result = runner.invoke(
+ evaluate,
+ [
+ config.name,
+ "-vv",
+ f"--output-folder={output_folder}",
+ "--predictions-folder=predictions",
+ f"--overlayed={overlay_folder}",
+ ],
+ )
+ _assert_exit_0(result)
+
+ assert os.path.exists(os.path.join(output_folder, "test.csv"))
+ assert os.path.exists(os.path.join(output_folder,
+ "second-annotator", "test.csv"))
+
+ # check overlayed images are there (since we requested them)
+ basedir = os.path.join(overlay_folder, "stare-images")
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
+
+ keywords = {
+ r"^Skipping dataset '__train__'": 0,
+ r"^Saving averages over all input images.*$": 2,
+ r"^Maximum F1-score of.*\(chosen \*a posteriori\*\)$": 1,
+ r"^F1-score of.*\(chosen \*a priori\*\)$": 1,
+ r"^F1-score of.*\(second annotator; threshold=0.5\)$": 1,
+ }
+ buf.seek(0)
+ logging_output = buf.read()
+
+ for k, v in keywords.items():
+ # if _str_counter(k, logging_output) != v:
+ # print(f"Count for string '{k}' appeared " \
+ # f"({_str_counter(k, result.output)}) " \
+ # f"instead of the expected {v}")
+ assert _str_counter(k, logging_output) == v, (
+ f"Count for string '{k}' appeared "
+ f"({_str_counter(k, logging_output)}) "
+ f"instead of the expected {v}:\nOutput:\n{logging_output}"
+ )
+
+
+def _check_compare(runner):
+
+ from ..script.compare import compare
+
+ with stdout_logging() as buf:
+
+ output_folder = "evaluations"
+ result = runner.invoke(
+ compare,
+ [
+ "-vv",
+ # label - path to metrics
+ "test",
+ os.path.join(output_folder, "test.csv"),
+ "test (2nd. human)",
+ os.path.join(output_folder, "second-annotator", "test.csv"),
+ "--output-figure=comparison.pdf",
+ "--output-table=comparison.rst",
+ ],
+ )
+ _assert_exit_0(result)
+
+ assert os.path.exists("comparison.pdf")
+ assert os.path.exists("comparison.rst")
+
+ keywords = {
+ r"^Loading metrics from": 2,
+ r"^Creating and saving plot at": 1,
+ r"^Tabulating performance summary...": 1,
+ r"^Saving table at": 1,
+ }
+ buf.seek(0)
+ logging_output = buf.read()
+
+ for k, v in keywords.items():
+ # if _str_counter(k, logging_output) != v:
+ # print(f"Count for string '{k}' appeared " \
+ # f"({_str_counter(k, result.output)}) " \
+ # f"instead of the expected {v}")
+ assert _str_counter(k, logging_output) == v, (
+ f"Count for string '{k}' appeared "
+ f"({_str_counter(k, logging_output)}) "
+ f"instead of the expected {v}:\nOutput:\n{logging_output}"
+ )
+
+
+@rc_variable_set("bob.ip.binseg.stare.datadir")
+def test_discrete_experiment_stare():
+
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ _check_train(runner)
+ _check_predict(runner)
+ _check_evaluate(runner)
+ _check_compare(runner)
+
+
+def test_train_help():
+ from ..script.train import train
+
+ _check_help(train)
+
+
+def test_predict_help():
+ from ..script.predict import predict
+
+ _check_help(predict)
+
+
+def test_evaluate_help():
+ from ..script.evaluate import evaluate
+
+ _check_help(evaluate)
+
+
+def test_compare_help():
+ from ..script.compare import compare
+
+ _check_help(compare)
+
+
+def test_config_help():
+ from ..script.config import config
+
+ _check_help(config)
+
+
+def test_config_list_help():
+ from ..script.config import list
+
+ _check_help(list)
+
+
+def test_config_list():
+ from ..script.config import list
+
+ runner = CliRunner()
+ result = runner.invoke(list)
+ _assert_exit_0(result)
+ assert "module: bob.ip.binseg.configs.datasets" in result.output
+ assert "module: bob.ip.binseg.configs.models" in result.output
+
+
+def test_config_list_v():
+ from ..script.config import list
+
+ runner = CliRunner()
+ result = runner.invoke(list, ["--verbose"])
+ _assert_exit_0(result)
+ assert "module: bob.ip.binseg.configs.datasets" in result.output
+ assert "module: bob.ip.binseg.configs.models" in result.output
+
+
+def test_config_describe_help():
+ from ..script.config import describe
+
+ _check_help(describe)
+
+
+def test_config_describe_drive():
+ from ..script.config import describe
+
+ runner = CliRunner()
+ result = runner.invoke(describe, ["drive"])
+ _assert_exit_0(result)
+ assert "[DRIVE-2004]" in result.output
+
+
+def test_config_copy_help():
+ from ..script.config import copy
+
+ _check_help(copy)
+
+
+def test_config_copy():
+ from ..script.config import copy
+
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ result = runner.invoke(copy, ["drive", "test.py"])
+ _assert_exit_0(result)
+ with open("test.py") as f:
+ data = f.read()
+ assert "[DRIVE-2004]" in data
+
+
+def test_dataset_help():
+ from ..script.dataset import dataset
+
+ _check_help(dataset)
+
+
+def test_dataset_list_help():
+ from ..script.dataset import list
+
+ _check_help(list)
+
+
+def test_dataset_list():
+ from ..script.dataset import list
+
+ runner = CliRunner()
+ result = runner.invoke(list)
+ _assert_exit_0(result)
+ assert result.output.startswith("Supported datasets:")
+
+
+def test_dataset_check_help():
+ from ..script.dataset import check
+
+ _check_help(check)
+
+
+def test_dataset_check():
+ from ..script.dataset import check
+
+ runner = CliRunner()
+ result = runner.invoke(check, ["--verbose", "--verbose", "--limit=2"])
+ _assert_exit_0(result)
diff --git a/bob/ip/binseg/test/test_config.py b/bob/ip/binseg/test/test_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..52e3a2b3258485e752cb0887b80bb72e6f05cda4
--- /dev/null
+++ b/bob/ip/binseg/test/test_config.py
@@ -0,0 +1,555 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import importlib
+
+import nose.tools
+
+import torch
+
+from . import mock_dataset
+stare_datadir, stare_dataset, stare_variable_set = mock_dataset()
+from .utils import rc_variable_set
+
+# we only iterate over the first N elements at most - dataset loading has
+# already been checked on the individual datset tests. Here, we are only
+# testing for the extra tools wrapping the dataset
+N = 10
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+def test_drive():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples:
+ nose.tools.eq_(len(s), 4)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+ nose.tools.eq_(s[3].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(s[3].dtype, torch.float32)
+
+ from ..configs.datasets.drive.default import dataset
+
+ nose.tools.eq_(len(dataset), 3)
+ _check_subset(dataset["__train__"], 20)
+ _check_subset(dataset["train"], 20)
+ _check_subset(dataset["test"], 20)
+
+ from ..configs.datasets.drive.second_annotator import dataset
+
+ nose.tools.eq_(len(dataset), 1)
+ _check_subset(dataset["test"], 20)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_drive_mtest():
+
+ from ..configs.datasets.drive.mtest import dataset
+ nose.tools.eq_(len(dataset), 6)
+
+ from ..configs.datasets.drive.default import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ for subset in dataset:
+ for sample in dataset[subset]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 544, 544))
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 544, 544))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_drive_covd():
+
+ from ..configs.datasets.drive.covd import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.drive.default import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 53)
+
+ for sample in dataset["__train__"]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_drive_ssl():
+
+ from ..configs.datasets.drive.ssl import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.drive.default import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 53)
+
+ for sample in dataset["__train__"]:
+ assert 5 <= len(sample) <= 6
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 6:
+ nose.tools.eq_(sample[3].shape, (1, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+ assert isinstance(sample[4], str)
+ nose.tools.eq_(sample[5].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[5].dtype, torch.float32)
+ else:
+ assert isinstance(sample[3], str)
+ nose.tools.eq_(sample[4].shape, (3, 544, 544)) #planes, height, width
+ nose.tools.eq_(sample[4].dtype, torch.float32)
+
+
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+def test_stare_augmentation_manipulation():
+
+ # some tests to check our context management for dataset augmentation works
+ # adequately, with one example dataset
+
+ # hack to allow testing on the CI
+ from ..configs.datasets.stare import _maker
+ dataset = _maker("ah", stare_dataset)
+
+ nose.tools.eq_(len(dataset["__train__"]._transforms.transforms),
+ len(dataset["test"]._transforms.transforms) + 4)
+
+ nose.tools.eq_(len(dataset["train"]._transforms.transforms),
+ len(dataset["test"]._transforms.transforms))
+
+
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+def test_stare():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 608, 704)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 608, 704)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ # hack to allow testing on the CI
+ from ..configs.datasets.stare import _maker
+
+ for protocol in "ah", "vk":
+ dataset = _maker(protocol, stare_dataset)
+ nose.tools.eq_(len(dataset), 3)
+ _check_subset(dataset["__train__"], 10)
+ _check_subset(dataset["train"], 10)
+ _check_subset(dataset["test"], 10)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_stare_mtest():
+
+ from ..configs.datasets.stare.mtest import dataset
+ nose.tools.eq_(len(dataset), 6)
+
+ from ..configs.datasets.stare.ah import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ for subset in dataset:
+ for sample in dataset[subset]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 608, 704)) #planes,height,width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 608, 704)) #planes,height,width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 608, 704))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_stare_covd():
+
+ from ..configs.datasets.stare.covd import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.stare.ah import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 63)
+ for sample in dataset["__train__"]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 608, 704)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 608, 704)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 608, 704)) #planes, height, width
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+def test_chasedb1():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 960, 960)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 960, 960)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ for m in ("first_annotator", "second_annotator"):
+ d = importlib.import_module(f"...configs.datasets.chasedb1.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 3)
+ _check_subset(d["__train__"], 8)
+ _check_subset(d["train"], 8)
+ _check_subset(d["test"], 20)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_chasedb1_mtest():
+
+ from ..configs.datasets.chasedb1.mtest import dataset
+ nose.tools.eq_(len(dataset), 6)
+
+ from ..configs.datasets.chasedb1.first_annotator import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ for subset in dataset:
+ for sample in dataset[subset]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 960, 960)) #planes,height,width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 960, 960)) #planes,height,width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 960, 960))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_chasedb1_covd():
+
+ from ..configs.datasets.chasedb1.covd import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.chasedb1.first_annotator import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 65)
+ for sample in dataset["__train__"]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 960, 960)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 960, 960)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 960, 960)) #planes, height, width
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+def test_hrf():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples:
+ nose.tools.eq_(len(s), 4)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 1168, 1648)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 1168, 1648)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+ nose.tools.eq_(s[3].shape, (1, 1168, 1648)) #planes, height, width
+ nose.tools.eq_(s[3].dtype, torch.float32)
+
+ from ..configs.datasets.hrf.default import dataset
+ nose.tools.eq_(len(dataset), 3)
+ _check_subset(dataset["__train__"], 15)
+ _check_subset(dataset["train"], 15)
+ _check_subset(dataset["test"], 30)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_hrf_mtest():
+
+ from ..configs.datasets.hrf.mtest import dataset
+ nose.tools.eq_(len(dataset), 6)
+
+ from ..configs.datasets.hrf.default import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ for subset in dataset:
+ for sample in dataset[subset]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 1168, 1648)) #planes,height,width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 1168, 1648)) #planes,height,width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 1168, 1648))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_hrf_covd():
+
+ from ..configs.datasets.hrf.covd import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.hrf.default import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 58)
+ for sample in dataset["__train__"]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 1168, 1648)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 1168, 1648)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 1168, 1648))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_iostar():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples:
+ nose.tools.eq_(len(s), 4)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 1024, 1024)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 1024, 1024)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+ nose.tools.eq_(s[3].shape, (1, 1024, 1024)) #planes, height, width
+ nose.tools.eq_(s[3].dtype, torch.float32)
+
+ for m in ("vessel", "optic_disc"):
+ d = importlib.import_module(f"...configs.datasets.iostar.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 3)
+ _check_subset(d["__train__"], 20)
+ _check_subset(d["train"], 20)
+ _check_subset(d["test"], 10)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_iostar_mtest():
+
+ from ..configs.datasets.iostar.vessel_mtest import dataset
+ nose.tools.eq_(len(dataset), 6)
+
+ from ..configs.datasets.iostar.vessel import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ for subset in dataset:
+ for sample in dataset[subset]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 1024, 1024)) #planes,height,width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 1024, 1024)) #planes,height,width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 1024, 1024))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.drive.datadir")
+@stare_variable_set("bob.ip.binseg.stare.datadir")
+@rc_variable_set("bob.ip.binseg.chasedb1.datadir")
+@rc_variable_set("bob.ip.binseg.hrf.datadir")
+@rc_variable_set("bob.ip.binseg.iostar.datadir")
+def test_iostar_covd():
+
+ from ..configs.datasets.iostar.covd import dataset
+ nose.tools.eq_(len(dataset), 3)
+
+ from ..configs.datasets.iostar.vessel import dataset as baseline
+ nose.tools.eq_(dataset["train"], baseline["train"])
+ nose.tools.eq_(dataset["test"], baseline["test"])
+
+ # this is the only different set from the baseline
+ nose.tools.eq_(len(dataset["__train__"]), 53)
+ for sample in dataset["__train__"]:
+ assert 3 <= len(sample) <= 4
+ assert isinstance(sample[0], str)
+ nose.tools.eq_(sample[1].shape, (3, 1024, 1024)) #planes, height, width
+ nose.tools.eq_(sample[1].dtype, torch.float32)
+ nose.tools.eq_(sample[2].shape, (1, 1024, 1024)) #planes, height, width
+ nose.tools.eq_(sample[2].dtype, torch.float32)
+ if len(sample) == 4:
+ nose.tools.eq_(sample[3].shape, (1, 1024, 1024))
+ nose.tools.eq_(sample[3].dtype, torch.float32)
+
+
+@rc_variable_set("bob.ip.binseg.refuge.datadir")
+def test_refuge():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples[:N]:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 1632, 1632)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 1632, 1632)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ for m in ("disc", "cup"):
+ d = importlib.import_module(f"...configs.datasets.refuge.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 4)
+ _check_subset(d["__train__"], 400)
+ _check_subset(d["train"], 400)
+ _check_subset(d["validation"], 400)
+ _check_subset(d["test"], 400)
+
+
+@rc_variable_set("bob.ip.binseg.drishtigs1.datadir")
+def test_drishtigs1():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples[:N]:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 1760, 2048)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 1760, 2048)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ for m in ("disc_all", "cup_all", "disc_any", "cup_any"):
+ d = importlib.import_module(f"...configs.datasets.drishtigs1.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 3)
+ _check_subset(d["__train__"], 50)
+ _check_subset(d["train"], 50)
+ _check_subset(d["test"], 51)
+
+
+@rc_variable_set("bob.ip.binseg.rimoner3.datadir")
+def test_rimoner3():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples[:N]:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 1440, 1088)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 1440, 1088)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ for m in ("disc_exp1", "cup_exp1", "disc_exp2", "cup_exp2"):
+ d = importlib.import_module(f"...configs.datasets.rimoner3.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 3)
+ _check_subset(d["__train__"], 99)
+ _check_subset(d["train"], 99)
+ _check_subset(d["test"], 60)
+
+
+@rc_variable_set("bob.ip.binseg.drionsdb.datadir")
+def test_drionsdb():
+
+ def _check_subset(samples, size):
+ nose.tools.eq_(len(samples), size)
+ for s in samples[:N]:
+ nose.tools.eq_(len(s), 3)
+ assert isinstance(s[0], str)
+ nose.tools.eq_(s[1].shape, (3, 416, 608)) #planes, height, width
+ nose.tools.eq_(s[1].dtype, torch.float32)
+ nose.tools.eq_(s[2].shape, (1, 416, 608)) #planes, height, width
+ nose.tools.eq_(s[2].dtype, torch.float32)
+
+ for m in ("expert1", "expert2"):
+ d = importlib.import_module(f"...configs.datasets.drionsdb.{m}",
+ package=__name__).dataset
+ nose.tools.eq_(len(d), 3)
+ _check_subset(d["__train__"], 60)
+ _check_subset(d["train"], 60)
+ _check_subset(d["test"], 50)
diff --git a/bob/ip/binseg/test/test_dataset.py b/bob/ip/binseg/test/test_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..f06d1a4ab2fe6d313af6d65ba5d81529daa6ca71
--- /dev/null
+++ b/bob/ip/binseg/test/test_dataset.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Test code for datasets"""
+
+import os
+import pkg_resources
+import nose.tools
+
+from ..data.dataset import CSVDataset, JSONDataset
+from ..data.sample import Sample
+
+
+def _data_file(f):
+ return pkg_resources.resource_filename(__name__, os.path.join("data", f))
+
+
+def _raw_data_loader(context, d):
+ return Sample(
+ data=[
+ float(d["sepal_length"]),
+ float(d["sepal_width"]),
+ float(d["petal_length"]),
+ float(d["petal_width"]),
+ d["species"][5:],
+ ],
+ key=(context["subset"] + str(context["order"]))
+ )
+
+
+def test_csv_loading():
+
+ # tests if we can build a simple CSV loader for the Iris Flower dataset
+ subsets = {
+ "train": _data_file("iris-train.csv"),
+ "test": _data_file("iris-train.csv")
+ }
+
+ fieldnames = (
+ "sepal_length",
+ "sepal_width",
+ "petal_length",
+ "petal_width",
+ "species",
+ )
+
+ dataset = CSVDataset(subsets, fieldnames, _raw_data_loader)
+ dataset.check()
+
+ data = dataset.subsets()
+
+ nose.tools.eq_(len(data["train"]), 75)
+ for k in data["train"]:
+ for f in range(4):
+ nose.tools.eq_(type(k.data[f]), float)
+ nose.tools.eq_(type(k.data[4]), str)
+ nose.tools.eq_(type(k.key), str)
+
+ nose.tools.eq_(len(data["test"]), 75)
+ for k in data["test"]:
+ for f in range(4):
+ nose.tools.eq_(type(k.data[f]), float)
+ nose.tools.eq_(type(k.data[4]), str)
+ assert k.data[4] in ("setosa", "versicolor", "virginica")
+ nose.tools.eq_(type(k.key), str)
+
+
+def test_json_loading():
+
+ # tests if we can build a simple JSON loader for the Iris Flower dataset
+ protocols = {"default": _data_file("iris.json")}
+
+ fieldnames = (
+ "sepal_length",
+ "sepal_width",
+ "petal_length",
+ "petal_width",
+ "species",
+ )
+
+ dataset = JSONDataset(protocols, fieldnames, _raw_data_loader)
+ dataset.check()
+
+ data = dataset.subsets("default")
+
+ nose.tools.eq_(len(data["train"]), 75)
+ for k in data["train"]:
+ for f in range(4):
+ nose.tools.eq_(type(k.data[f]), float)
+ nose.tools.eq_(type(k.data[4]), str)
+ nose.tools.eq_(type(k.key), str)
+
+ nose.tools.eq_(len(data["test"]), 75)
+ for k in data["test"]:
+ for f in range(4):
+ nose.tools.eq_(type(k.data[f]), float)
+ nose.tools.eq_(type(k.data[4]), str)
+ nose.tools.eq_(type(k.key), str)
diff --git a/bob/ip/binseg/test/test_drionsdb.py b/bob/ip/binseg/test/test_drionsdb.py
new file mode 100644
index 0000000000000000000000000000000000000000..7508a23df9a5bc41f13a814584a431c2e9cc4a94
--- /dev/null
+++ b/bob/ip/binseg/test/test_drionsdb.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for DRIONS-DB"""
+
+import os
+
+import numpy
+import nose.tools
+from nose.plugins.attrib import attr
+
+from ..data.drionsdb import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ for protocol in ("expert1", "expert2"):
+
+ subset = dataset.subsets(protocol)
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 60)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("images", "image_0"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 50)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("images", "image_"))
+
+
+@rc_variable_set("bob.ip.binseg.drionsdb.datadir")
+@attr("slow")
+def test_loading():
+
+ image_size = (600, 400)
+
+ def _check_sample(s, bw_threshold_label):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 2)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+
+ b, w = count_bw(data["label"])
+ assert (b + w) == numpy.prod(image_size), (
+ f"Counts of black + white ({b}+{w}) do not add up to total "
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ )
+ assert (w / b) < bw_threshold_label, (
+ f"The proportion between black and white pixels "
+ f"({w}/{b}={w/b:.3f}) is larger than the allowed threshold "
+ f"of {bw_threshold_label} at '{s.key}':label - this could "
+ f"indicate a loading problem!"
+ )
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("expert1")
+ proportions = [_check_sample(s, 0.046) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.043) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("expert2")
+ proportions = [_check_sample(s, 0.044) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.045) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+@rc_variable_set("bob.ip.binseg.drionsdb.datadir")
+@attr("slow")
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_drishtigs1.py b/bob/ip/binseg/test/test_drishtigs1.py
new file mode 100644
index 0000000000000000000000000000000000000000..17e39a2ad455b6f04ed5f9d4798d581bc4b668a1
--- /dev/null
+++ b/bob/ip/binseg/test/test_drishtigs1.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for Drishti-GS1"""
+
+import os
+
+import numpy
+import nose.tools
+from nose.plugins.attrib import attr
+
+from ..data.drishtigs1 import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ for protocol in ("optic-disc-all", "optic-cup-all", "optic-disc-any",
+ "optic-cup-any"):
+
+ subset = dataset.subsets(protocol)
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 50)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("Drishti-GS1_files",
+ "Training", "Images", "drishtiGS_"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 51)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("Drishti-GS1_files",
+ "Test", "Images", "drishtiGS_"))
+
+
+@rc_variable_set("bob.ip.binseg.drishtigs1.datadir")
+@attr("slow")
+def test_loading():
+
+ def _check_sample(s, bw_threshold_label):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 2)
+
+ assert "data" in data
+ assert data["data"].size[0] > 2040, (
+ f"Width ({data['data'].size[0]}) for {s.key} is smaller "
+ f"than 2040 pixels"
+ )
+ assert data["data"].size[1] > 1740, (
+ f"Width ({data['data'].size[1]}) for {s.key} is smaller "
+ f"than 1740 pixels"
+ )
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ #nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["data"].size, data["label"].size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b + w) == numpy.prod(data["data"].size), (
+ f"Counts of black + white ({b}+{w}) do not add up to total "
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ )
+ assert (w / b) < bw_threshold_label, (
+ f"The proportion between black and white pixels "
+ f"({w}/{b}={w/b:.3f}) is larger than the allowed threshold "
+ f"of {bw_threshold_label} at '{s.key}':label - this could "
+ f"indicate a loading problem!"
+ )
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ limit = None
+ subset = dataset.subsets("optic-cup-all")
+ proportions = [_check_sample(s, 0.027) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.035) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-disc-all")
+ proportions = [_check_sample(s, 0.045) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.055) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-cup-any")
+ proportions = [_check_sample(s, 0.034) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.047) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-disc-any")
+ proportions = [_check_sample(s, 0.052) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.060) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+
+@rc_variable_set("bob.ip.binseg.drishtigs1.datadir")
+@attr("slow")
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_drive.py b/bob/ip/binseg/test/test_drive.py
new file mode 100644
index 0000000000000000000000000000000000000000..53c953e9fd3d2e54785129bb57c674800cd742d3
--- /dev/null
+++ b/bob/ip/binseg/test/test_drive.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for DRIVE"""
+
+import os
+
+import numpy
+import nose.tools
+
+from ..data.drive import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ subset = dataset.subsets("default")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 20)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("training", "images"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 20)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("test", "images"))
+
+ subset = dataset.subsets("second-annotator")
+ nose.tools.eq_(len(subset), 1)
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 20)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("test", "images"))
+
+
+@rc_variable_set('bob.ip.binseg.drive.datadir')
+def test_loading():
+
+ image_size = (565, 584)
+
+ def _check_sample(s, bw_threshold_label, bw_threshold_mask):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 3)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels in labels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ assert "mask" in data
+ nose.tools.eq_(data["mask"].size, image_size)
+ nose.tools.eq_(data["mask"].mode, "1")
+ bm, wm = count_bw(data["mask"])
+ assert (bm+wm) == numpy.prod(image_size), \
+ f"Counts of black + white ({bm}+{wm}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
+ assert (wm/bm) > bw_threshold_mask, \
+ f"The proportion between black and white pixels in masks " \
+ f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+ f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
+ f"this could indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels and blue area indicating the
+ # parts to be masked out.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"], data["mask"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b, wm/bm
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("default")
+ proportions = [_check_sample(s, 0.14, 2.14) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+ proportions = [_check_sample(s, 0.12, 2.12) for s in subset["test"]][:limit]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+
+ subset = dataset.subsets("second-annotator")
+ proportions = [_check_sample(s, 0.12, 2.12) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+
+
+@rc_variable_set('bob.ip.binseg.drive.datadir')
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_hrf.py b/bob/ip/binseg/test/test_hrf.py
new file mode 100644
index 0000000000000000000000000000000000000000..fdddfbbc78d79c228e1bca01e3a3ebf9dda84f65
--- /dev/null
+++ b/bob/ip/binseg/test/test_hrf.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for HRF"""
+
+import os
+
+import numpy
+import nose.tools
+
+from ..data.hrf import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ subset = dataset.subsets("default")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 15)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("images", "0"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 30)
+ for s in subset["test"]:
+ assert s.key.startswith("images")
+
+
+@rc_variable_set('bob.ip.binseg.hrf.datadir')
+def test_loading():
+
+ image_size = (3504, 2336)
+
+ def _check_sample(s, bw_threshold_label, bw_threshold_mask):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 3)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ assert "mask" in data
+ nose.tools.eq_(data["mask"].size, image_size)
+ nose.tools.eq_(data["mask"].mode, "1")
+ bm, wm = count_bw(data["mask"])
+ assert (bm+wm) == numpy.prod(image_size), \
+ f"Counts of black + white ({bm}+{wm}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
+ assert (wm/bm) > bw_threshold_mask, \
+ f"The proportion between black and white pixels in masks " \
+ f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+ f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
+ f"this could indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels and blue area indicating the
+ # parts to be masked out.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"], data["mask"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b, wm/bm
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("default")
+ proportions = [_check_sample(s, 0.12, 5.42) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+ proportions = [_check_sample(s, 0.12, 5.41) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+
+
+@rc_variable_set('bob.ip.binseg.hrf.datadir')
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_iostar.py b/bob/ip/binseg/test/test_iostar.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d8946b60d1160fa5b0dca95c39de7badac1a1aa
--- /dev/null
+++ b/bob/ip/binseg/test/test_iostar.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for IOSTAR"""
+
+import os
+
+import numpy
+import nose.tools
+
+from ..data.iostar import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ subset = dataset.subsets("vessel")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 20)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("image", "STAR "))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 10)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("image", "STAR "))
+
+ subset = dataset.subsets("optic-disc")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 20)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("image", "STAR "))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 10)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("image", "STAR "))
+
+
+@rc_variable_set('bob.ip.binseg.iostar.datadir')
+def test_loading():
+
+ image_size = (1024, 1024)
+
+ def _check_sample(s, bw_threshold_label, bw_threshold_mask):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 3)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ assert "mask" in data
+ nose.tools.eq_(data["mask"].size, image_size)
+ nose.tools.eq_(data["mask"].mode, "1")
+ bm, wm = count_bw(data["mask"])
+ assert (bm+wm) == numpy.prod(image_size), \
+ f"Counts of black + white ({bm}+{wm}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':mask"
+ assert (wm/bm) > bw_threshold_mask, \
+ f"The proportion between black and white pixels in masks " \
+ f"({wm}/{bm}={wm/bm:.2f}) is smaller than the allowed " \
+ f"threshold of {bw_threshold_mask} at '{s.key}':label - " \
+ f"this could indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels and blue area indicating the
+ # parts to be masked out.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"], data["mask"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b, wm/bm
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("vessel")
+ proportions = [_check_sample(s, 0.11, 3.19) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+ proportions = [_check_sample(s, 0.10, 3.27) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+
+ subset = dataset.subsets("optic-disc")
+ proportions = [_check_sample(s, 0.023, 3.19) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+ proportions = [_check_sample(s, 0.033, 3.27) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(k[0] for k in proportions)}")
+ #print(f"min mask proportions = {min(k[1] for k in proportions)}")
+
+@rc_variable_set('bob.ip.binseg.iostar.datadir')
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_refuge.py b/bob/ip/binseg/test/test_refuge.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4fac282cc210e1f04d09d74ad16d818641cae59
--- /dev/null
+++ b/bob/ip/binseg/test/test_refuge.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for REFUGE"""
+
+import os
+
+import numpy
+import nose.tools
+from nose.plugins.attrib import attr
+
+from ..data.refuge import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ for protocol in ("optic-disc", "optic-cup"):
+
+ subset = dataset.subsets(protocol)
+ nose.tools.eq_(len(subset), 3)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 400)
+ for s in subset["train"]:
+ assert s.key.startswith("Training400")
+
+ assert "validation" in subset
+ nose.tools.eq_(len(subset["validation"]), 400)
+ for s in subset["validation"]:
+ assert s.key.startswith("REFUGE-Validation400")
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 400)
+ for s in subset["test"]:
+ assert s.key.startswith("Test400")
+
+
+@rc_variable_set("bob.ip.binseg.refuge.datadir")
+@attr("slow")
+def test_loading():
+
+ def _check_sample(
+ s, image_size, glaucoma_label, entries, bw_threshold_label
+ ):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), entries)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b + w) == numpy.prod(image_size), (
+ f"Counts of black + white ({b}+{w}) do not add up to total "
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ )
+ assert (w / b) < bw_threshold_label, (
+ f"The proportion between black and white pixels "
+ f"({w}/{b}={w/b:.3f}) is larger than the allowed threshold "
+ f"of {bw_threshold_label} at '{s.key}':label - this could "
+ f"indicate a loading problem!"
+ )
+
+ if glaucoma_label:
+ assert "glaucoma" in data
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("optic-disc")
+ proportions = [_check_sample(s, (2124, 2056), True, 3, 0.029) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, (1634, 1634), False, 2, 0.043) for s in subset["validation"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, (1634, 1634), True, 3, 0.026) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-cup")
+ proportions = [_check_sample(s, (2124, 2056), True, 3, 0.018) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, (1634, 1634), False, 2, 0.030) for s in subset["validation"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, (1634, 1634), True, 3, 0.017) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+
+@rc_variable_set("bob.ip.binseg.refuge.datadir")
+@attr("slow")
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_rimoner3.py b/bob/ip/binseg/test/test_rimoner3.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6010cf6a3dec0c9b4c5c84dc0857b92c349f174
--- /dev/null
+++ b/bob/ip/binseg/test/test_rimoner3.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for RIM-ONE r3"""
+
+import os
+
+import numpy
+import nose.tools
+from nose.plugins.attrib import attr
+
+from ..data.rimoner3 import dataset
+from .utils import rc_variable_set, count_bw
+
+
+def test_protocol_consistency():
+
+ for protocol in ("optic-disc-exp1", "optic-cup-exp1", "optic-disc-exp2",
+ "optic-cup-exp2", "optic-disc-avg", "optic-cup-avg"):
+
+ subset = dataset.subsets(protocol)
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 99)
+ for s in subset["train"]:
+ assert "Stereo Images" in s.key
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 60)
+ for s in subset["test"]:
+ assert "Stereo Images" in s.key
+
+
+@rc_variable_set("bob.ip.binseg.rimoner3.datadir")
+@attr("slow")
+def test_loading():
+
+ image_size = (1072, 1424)
+
+ def _check_sample(s, bw_threshold_label):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 2)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ subset = dataset.subsets("optic-cup-exp1")
+ limit = None
+ proportions = [_check_sample(s, 0.048) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.042) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-disc-exp1")
+ proportions = [_check_sample(s, 0.088) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.061) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-cup-exp2")
+ proportions = [_check_sample(s, 0.039) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.038) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-disc-exp2")
+ proportions = [_check_sample(s, 0.090) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.065) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-cup-avg")
+ proportions = [_check_sample(s, 0.042) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.040) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("optic-disc-avg")
+ proportions = [_check_sample(s, 0.089) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.063) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+
+@rc_variable_set("bob.ip.binseg.rimoner3.datadir")
+@attr("slow")
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_stare.py b/bob/ip/binseg/test/test_stare.py
new file mode 100644
index 0000000000000000000000000000000000000000..edc52ae1cccb0d2efa6b00d7e0469898f4c1eef5
--- /dev/null
+++ b/bob/ip/binseg/test/test_stare.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Tests for STARE"""
+
+import os
+
+import numpy
+import nose.tools
+
+## special trick for CI builds
+from . import mock_dataset
+datadir, dataset, rc_variable_set = mock_dataset()
+
+from .utils import count_bw
+
+
+def test_protocol_consistency():
+
+ subset = dataset.subsets("ah")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 10)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("stare-images", "im0"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 10)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("stare-images", "im0"))
+
+ subset = dataset.subsets("vk")
+ nose.tools.eq_(len(subset), 2)
+
+ assert "train" in subset
+ nose.tools.eq_(len(subset["train"]), 10)
+ for s in subset["train"]:
+ assert s.key.startswith(os.path.join("stare-images", "im0"))
+
+ assert "test" in subset
+ nose.tools.eq_(len(subset["test"]), 10)
+ for s in subset["test"]:
+ assert s.key.startswith(os.path.join("stare-images", "im0"))
+
+
+@rc_variable_set('bob.ip.binseg.stare.datadir')
+def test_loading():
+
+ image_size = (700, 605)
+
+ def _check_sample(s, bw_threshold_label):
+
+ data = s.data
+ assert isinstance(data, dict)
+ nose.tools.eq_(len(data), 2)
+
+ assert "data" in data
+ nose.tools.eq_(data["data"].size, image_size)
+ nose.tools.eq_(data["data"].mode, "RGB")
+
+ assert "label" in data
+ nose.tools.eq_(data["label"].size, image_size)
+ nose.tools.eq_(data["label"].mode, "1")
+ b, w = count_bw(data["label"])
+ assert (b+w) == numpy.prod(image_size), \
+ f"Counts of black + white ({b}+{w}) do not add up to total " \
+ f"image size ({numpy.prod(image_size)}) at '{s.key}':label"
+ assert (w/b) < bw_threshold_label, \
+ f"The proportion between black and white pixels " \
+ f"({w}/{b}={w/b:.2f}) is larger than the allowed threshold " \
+ f"of {bw_threshold_label} at '{s.key}':label - this could " \
+ f"indicate a loading problem!"
+
+ # to visualize images, uncomment the folowing code
+ # it should display an image with a faded background representing the
+ # original data, blended with green labels.
+ #from ..data.utils import overlayed_image
+ #display = overlayed_image(data["data"], data["label"])
+ #display.show()
+ #import ipdb; ipdb.set_trace()
+
+ return w/b
+
+ limit = None #use this to limit testing to first images only
+ subset = dataset.subsets("ah")
+ proportions = [_check_sample(s, 0.10) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.12) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+ subset = dataset.subsets("vk")
+ proportions = [_check_sample(s, 0.19) for s in subset["train"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+ proportions = [_check_sample(s, 0.18) for s in subset["test"][:limit]]
+ #print(f"max label proportions = {max(proportions)}")
+
+
+@rc_variable_set('bob.ip.binseg.stare.datadir')
+def test_check():
+ nose.tools.eq_(dataset.check(), 0)
diff --git a/bob/ip/binseg/test/test_summary.py b/bob/ip/binseg/test/test_summary.py
index 7faabf796674db6b7914d631ba41f9160c08a623..a6d9948aed57612f238ec3a1333e7148b5401cb7 100644
--- a/bob/ip/binseg/test/test_summary.py
+++ b/bob/ip/binseg/test/test_summary.py
@@ -1,9 +1,8 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import torch
+import os
import unittest
-import numpy as np
from bob.ip.binseg.modeling.driu import build_driu
from bob.ip.binseg.modeling.driuod import build_driuod
from bob.ip.binseg.modeling.hed import build_hed
@@ -11,36 +10,42 @@ from bob.ip.binseg.modeling.unet import build_unet
from bob.ip.binseg.modeling.resunet import build_res50unet
from bob.ip.binseg.utils.summary import summary
+
class Tester(unittest.TestCase):
"""
Unit test for model architectures
- """
+ """
+
def test_summary_driu(self):
model = build_driu()
- param = summary(model)
- self.assertIsInstance(param,int)
+ s, param = summary(model)
+ self.assertIsInstance(s, str)
+ self.assertIsInstance(param, int)
-
- def test__summary_driuod(self):
+ def test_summary_driuod(self):
model = build_driuod()
- param = summary(model)
- self.assertIsInstance(param,int)
-
+ s, param = summary(model)
+ self.assertIsInstance(s, str)
+ self.assertIsInstance(param, int)
def test_summary_hed(self):
model = build_hed()
- param = summary(model)
- self.assertIsInstance(param,int)
+ s, param = summary(model)
+ self.assertIsInstance(s, str)
+ self.assertIsInstance(param, int)
def test_summary_unet(self):
model = build_unet()
- param = summary(model)
- self.assertIsInstance(param,int)
+ s, param = summary(model)
+ self.assertIsInstance(s, str)
+ self.assertIsInstance(param, int)
def test_summary_resunet(self):
model = build_res50unet()
- param = summary(model)
- self.assertIsInstance(param,int)
+ s, param = summary(model)
+ self.assertIsInstance(s, str)
+ self.assertIsInstance(param, int)
+
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+if __name__ == "__main__":
+ unittest.main()
diff --git a/bob/ip/binseg/test/test_transforms.py b/bob/ip/binseg/test/test_transforms.py
index 479cd79c063e8059c6048dfd0ead1f3a7062e006..e71a4a4927f1598a49037dc09a672de91845dd11 100644
--- a/bob/ip/binseg/test/test_transforms.py
+++ b/bob/ip/binseg/test/test_transforms.py
@@ -1,49 +1,365 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import os
+import random
+
+import nose.tools
+import pkg_resources
+
+import numpy
+import PIL.Image
import torch
-import unittest
-import numpy as np
-from bob.ip.binseg.data.transforms import *
-
-transforms = Compose([
- RandomHFlip(prob=1)
- ,RandomHFlip(prob=1)
- ,RandomVFlip(prob=1)
- ,RandomVFlip(prob=1)
- ])
-
-def create_img():
- t = torch.randn((3,42,24))
- pil = VF.to_pil_image(t)
+import torchvision.transforms.functional
+
+from ..data.transforms import *
+
+
+def _create_img(size):
+ t = torch.randn(size)
+ pil = torchvision.transforms.functional.to_pil_image(t)
return pil
-class Tester(unittest.TestCase):
- """
- Unit test for random flips
- """
-
- def test_flips(self):
- transforms = Compose([
- RandomHFlip(prob=1)
- ,RandomHFlip(prob=1)
- ,RandomVFlip(prob=1)
- ,RandomVFlip(prob=1)
- ])
- img, gt, mask = [create_img() for i in range(3)]
- img_t, gt_t, mask_t = transforms(img, gt, mask)
- self.assertTrue(np.all(np.array(img_t) == np.array(img)))
- self.assertTrue(np.all(np.array(gt_t) == np.array(gt)))
- self.assertTrue(np.all(np.array(mask_t) == np.array(mask)))
-
- def test_to_tensor(self):
- transforms = ToTensor()
- img, gt, mask = [create_img() for i in range(3)]
- img_t, gt_t, mask_t = transforms(img, gt, mask)
- self.assertEqual(str(img_t.dtype),"torch.float32")
- self.assertEqual(str(gt_t.dtype),"torch.float32")
- self.assertEqual(str(mask_t.dtype),"torch.float32")
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+def test_center_crop():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ crop_size = (10, 12) # (height, width)
+
+ # test
+ bh = (im_size[1] - crop_size[0]) // 2
+ bw = (im_size[2] - crop_size[1]) // 2
+ idx = (slice(bh, -bh), slice(bw, -bw), slice(0, im_size[0]))
+ transforms = CenterCrop(crop_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ nose.tools.eq_(
+ img_t.size, (crop_size[1], crop_size[0])
+ ) # confirms the above
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t) == numpy.array(img)[idx])
+ assert numpy.all(numpy.array(gt_t) == numpy.array(gt)[idx])
+ assert numpy.all(numpy.array(mask_t) == numpy.array(mask)[idx])
+
+
+def test_center_crop_uneven():
+
+ # parameters
+ im_size = (3, 23, 20) # (planes, height, width)
+ crop_size = (10, 13) # (height, width)
+
+ # test
+ bh = (im_size[1] - crop_size[0]) // 2
+ bw = (im_size[2] - crop_size[1]) // 2
+ # when the crop size is uneven, this is what happens - notice here that the
+ # image height is uneven, and the crop width as well - the attributions of
+ # extra pixels will depend on what is uneven (original image or crop)
+ idx = (slice(bh, -(bh + 1)), slice((bw + 1), -bw), slice(0, im_size[0]))
+ transforms = CenterCrop(crop_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ nose.tools.eq_(
+ img_t.size, (crop_size[1], crop_size[0])
+ ) # confirms the above
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t) == numpy.array(img)[idx])
+ assert numpy.all(numpy.array(gt_t) == numpy.array(gt)[idx])
+ assert numpy.all(numpy.array(mask_t) == numpy.array(mask)[idx])
+
+
+def test_pad_default():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ pad_size = 2
+
+ # test
+ idx = (
+ slice(pad_size, -pad_size),
+ slice(pad_size, -pad_size),
+ slice(0, im_size[0]),
+ )
+ transforms = Pad(pad_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t)[idx] == numpy.array(img))
+ assert numpy.all(numpy.array(gt_t)[idx] == numpy.array(gt))
+ assert numpy.all(numpy.array(mask_t)[idx] == numpy.array(mask))
+
+ # checks that the border introduced with padding is all about "fill"
+ img_t = numpy.array(img_t)
+ img_t[idx] = 0
+ border_size_plane = img_t[:, :, 0].size - numpy.array(img)[:, :, 0].size
+ nose.tools.eq_(img_t.sum(), 0)
+
+ gt_t = numpy.array(gt_t)
+ gt_t[idx] = 0
+ nose.tools.eq_(gt_t.sum(), 0)
+
+ mask_t = numpy.array(mask_t)
+ mask_t[idx] = 0
+ nose.tools.eq_(mask_t.sum(), 0)
+
+
+def test_pad_2tuple():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ pad_size = (1, 2) # left/right, top/bottom
+ fill = (3, 4, 5)
+
+ # test
+ idx = (
+ slice(pad_size[1], -pad_size[1]),
+ slice(pad_size[0], -pad_size[0]),
+ slice(0, im_size[0]),
+ )
+ transforms = Pad(pad_size, fill)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t)[idx] == numpy.array(img))
+ assert numpy.all(numpy.array(gt_t)[idx] == numpy.array(gt))
+ assert numpy.all(numpy.array(mask_t)[idx] == numpy.array(mask))
+
+ # checks that the border introduced with padding is all about "fill"
+ img_t = numpy.array(img_t)
+ img_t[idx] = 0
+ border_size_plane = img_t[:, :, 0].size - numpy.array(img)[:, :, 0].size
+ expected_sum = sum((fill[k] * border_size_plane) for k in range(3))
+ nose.tools.eq_(img_t.sum(), expected_sum)
+
+ gt_t = numpy.array(gt_t)
+ gt_t[idx] = 0
+ nose.tools.eq_(gt_t.sum(), expected_sum)
+
+ mask_t = numpy.array(mask_t)
+ mask_t[idx] = 0
+ nose.tools.eq_(mask_t.sum(), expected_sum)
+
+
+def test_pad_4tuple():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ pad_size = (1, 2, 3, 4) # left, top, right, bottom
+ fill = (3, 4, 5)
+
+ # test
+ idx = (
+ slice(pad_size[1], -pad_size[3]),
+ slice(pad_size[0], -pad_size[2]),
+ slice(0, im_size[0]),
+ )
+ transforms = Pad(pad_size, fill)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t)[idx] == numpy.array(img))
+ assert numpy.all(numpy.array(gt_t)[idx] == numpy.array(gt))
+ assert numpy.all(numpy.array(mask_t)[idx] == numpy.array(mask))
+
+ # checks that the border introduced with padding is all about "fill"
+ img_t = numpy.array(img_t)
+ img_t[idx] = 0
+ border_size_plane = img_t[:, :, 0].size - numpy.array(img)[:, :, 0].size
+ expected_sum = sum((fill[k] * border_size_plane) for k in range(3))
+ nose.tools.eq_(img_t.sum(), expected_sum)
+
+ gt_t = numpy.array(gt_t)
+ gt_t[idx] = 0
+ nose.tools.eq_(gt_t.sum(), expected_sum)
+
+ mask_t = numpy.array(mask_t)
+ mask_t[idx] = 0
+ nose.tools.eq_(mask_t.sum(), expected_sum)
+
+
+def test_resize_downscale_w():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ new_size = 10 # (smallest edge)
+
+ # test
+ transforms = Resize(new_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ new_size = (new_size, (new_size * im_size[1]) / im_size[2])
+ nose.tools.eq_(img_t.size, new_size)
+ nose.tools.eq_(gt_t.size, new_size)
+ nose.tools.eq_(mask_t.size, new_size)
+
+
+def test_resize_downscale_hw():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ new_size = (10, 12) # (height, width)
+
+ # test
+ transforms = Resize(new_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ nose.tools.eq_(img_t.size, (new_size[1], new_size[0]))
+ nose.tools.eq_(gt_t.size, (new_size[1], new_size[0]))
+ nose.tools.eq_(mask_t.size, (new_size[1], new_size[0]))
+
+
+def test_crop():
+
+ # parameters
+ im_size = (3, 22, 20) # (planes, height, width)
+ crop_size = (3, 2, 10, 12) # (upper, left, height, width)
+
+ # test
+ idx = (
+ slice(crop_size[0], crop_size[0] + crop_size[2]),
+ slice(crop_size[1], crop_size[1] + crop_size[3]),
+ slice(0, im_size[0]),
+ )
+ transforms = Crop(*crop_size)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ nose.tools.eq_(img.size, (im_size[2], im_size[1])) # confirms the above
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.array(img_t) == numpy.array(img)[idx])
+ assert numpy.all(numpy.array(gt_t) == numpy.array(gt)[idx])
+ assert numpy.all(numpy.array(mask_t) == numpy.array(mask)[idx])
+
+
+def test_to_tensor():
+
+ transforms = ToTensor()
+ img, gt, mask = [_create_img((3, 5, 5)) for i in range(3)]
+ gt = gt.convert("1", dither=None)
+ mask = mask.convert("1", dither=None)
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ nose.tools.eq_(img_t.dtype, torch.float32)
+ nose.tools.eq_(gt_t.dtype, torch.float32)
+ nose.tools.eq_(mask_t.dtype, torch.float32)
+
+
+def test_horizontal_flip():
+
+ transforms = RandomHorizontalFlip(p=1)
+
+ im_size = (3, 24, 42) # (planes, height, width)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.flip(img_t, axis=1) == numpy.array(img))
+ assert numpy.all(numpy.flip(gt_t, axis=1) == numpy.array(gt))
+ assert numpy.all(numpy.flip(mask_t, axis=1) == numpy.array(mask))
+
+
+def test_vertical_flip():
+
+ transforms = RandomVerticalFlip(p=1)
+
+ im_size = (3, 24, 42) # (planes, height, width)
+ img, gt, mask = [_create_img(im_size) for i in range(3)]
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+
+ # notice that PIL->array does array.transpose(1, 2, 0)
+ # so it creates an array that is (height, width, planes)
+ assert numpy.all(numpy.flip(img_t, axis=0) == numpy.array(img))
+ assert numpy.all(numpy.flip(gt_t, axis=0) == numpy.array(gt))
+ assert numpy.all(numpy.flip(mask_t, axis=0) == numpy.array(mask))
+
+
+def test_rotation():
+
+ im_size = (3, 24, 42) # (planes, height, width)
+ transforms = RandomRotation(degrees=90, p=1)
+ img = _create_img(im_size)
+
+ # asserts all images are rotated the same
+ # and they are different from the original
+ random.seed(42)
+ img1_t, img2_t, img3_t = transforms(img, img, img)
+ nose.tools.eq_(img1_t.size, (im_size[2], im_size[1]))
+ assert numpy.all(numpy.array(img1_t) == numpy.array(img2_t))
+ assert numpy.all(numpy.array(img1_t) == numpy.array(img3_t))
+ assert numpy.any(numpy.array(img1_t) != numpy.array(img))
+
+ # asserts two random transforms are not the same
+ (img_t2,) = transforms(img)
+ assert numpy.any(numpy.array(img_t2) != numpy.array(img1_t))
+
+
+def test_color_jitter():
+
+ im_size = (3, 24, 42) # (planes, height, width)
+ transforms = ColorJitter(p=1)
+ img = _create_img(im_size)
+
+ # asserts only the first image is jittered
+ # and it is different from the original
+ # all others match the input data
+ random.seed(42)
+ img1_t, img2_t, img3_t = transforms(img, img, img)
+ nose.tools.eq_(img1_t.size, (im_size[2], im_size[1]))
+ assert numpy.any(numpy.array(img1_t) != numpy.array(img))
+ assert numpy.any(numpy.array(img1_t) != numpy.array(img2_t))
+ assert numpy.all(numpy.array(img2_t) == numpy.array(img3_t))
+ assert numpy.all(numpy.array(img2_t) == numpy.array(img))
+
+ # asserts two random transforms are not the same
+ img1_t2, img2_t2, img3_t2 = transforms(img, img, img)
+ assert numpy.any(numpy.array(img1_t2) != numpy.array(img1_t))
+ assert numpy.all(numpy.array(img2_t2) == numpy.array(img))
+ assert numpy.all(numpy.array(img3_t2) == numpy.array(img))
+
+
+def test_compose():
+
+ transforms = Compose(
+ [
+ RandomVerticalFlip(p=1),
+ RandomHorizontalFlip(p=1),
+ RandomVerticalFlip(p=1),
+ RandomHorizontalFlip(p=1),
+ ]
+ )
+
+ img, gt, mask = [_create_img((3, 24, 42)) for i in range(3)]
+ img_t, gt_t, mask_t = transforms(img, gt, mask)
+ assert numpy.all(numpy.array(img_t) == numpy.array(img))
+ assert numpy.all(numpy.array(gt_t) == numpy.array(gt))
+ assert numpy.all(numpy.array(mask_t) == numpy.array(mask))
+
+
+def test_16bit_autolevel():
+
+ path = pkg_resources.resource_filename(__name__, os.path.join("data",
+ "img-16bit.png"))
+ # the way to load a 16-bit PNG image correctly, according to:
+ # https://stackoverflow.com/questions/32622658/read-16-bit-png-image-file-using-python
+ # https://github.com/python-pillow/Pillow/issues/3011
+ img = PIL.Image.fromarray(numpy.array(PIL.Image.open(path)).astype("uint16"))
+ nose.tools.eq_(img.mode, "I;16")
+ nose.tools.eq_(img.getextrema(), (0, 65281))
+
+ timg = SingleAutoLevel16to8()(img)
+ nose.tools.eq_(timg.mode, "L")
+ nose.tools.eq_(timg.getextrema(), (0, 255))
+ #timg.show()
+ #import ipdb; ipdb.set_trace()
diff --git a/bob/ip/binseg/test/utils.py b/bob/ip/binseg/test/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f2e7fcf62dc8404b88f75a15cb2ab8c96d4992a
--- /dev/null
+++ b/bob/ip/binseg/test/utils.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+"""Test utilities"""
+
+
+import functools
+
+import numpy
+import nose.plugins.skip
+
+import bob.extension
+
+
+def rc_variable_set(name):
+ """
+ Decorator that checks if a given bobrc variable is set before running
+ """
+
+ def wrapped_function(test):
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ if name not in bob.extension.rc:
+ raise nose.plugins.skip.SkipTest("Bob's RC variable '%s' is not set" % name)
+ return test(*args, **kwargs)
+
+ return wrapper
+
+ return wrapped_function
+
+
+def count_bw(b):
+ """Calculates totals of black and white pixels in a binary image
+
+
+ Parameters
+ ----------
+
+ b : PIL.Image.Image
+ A PIL image in mode "1" to be used for calculating positives and
+ negatives
+
+ Returns
+ -------
+
+ black : int
+ Number of black pixels in the binary image
+
+ white : int
+ Number of white pixels in the binary image
+ """
+
+ boolean_array = numpy.array(b)
+ white = boolean_array.sum()
+ return (boolean_array.size-white), white
diff --git a/bob/ip/binseg/utils/FreeMono.ttf b/bob/ip/binseg/utils/FreeMono.ttf
deleted file mode 100644
index 7485f9e4c84d5a372c81e11df2cd9f5e2eb2064a..0000000000000000000000000000000000000000
Binary files a/bob/ip/binseg/utils/FreeMono.ttf and /dev/null differ
diff --git a/bob/ip/binseg/utils/__init__.py b/bob/ip/binseg/utils/__init__.py
index 2ca5e07cb73f0bdddcb863ef497955964087e301..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/bob/ip/binseg/utils/__init__.py
+++ b/bob/ip/binseg/utils/__init__.py
@@ -1,3 +0,0 @@
-# see https://docs.python.org/3/library/pkgutil.html
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/checkpointer.py b/bob/ip/binseg/utils/checkpointer.py
index f3899e1dc2a23e6b6c8c250bebfd9a40ebc1fc93..4ae57e5c1a5f58727edc990f04e8553bc6f8de4e 100644
--- a/bob/ip/binseg/utils/checkpointer.py
+++ b/bob/ip/binseg/utils/checkpointer.py
@@ -1,23 +1,22 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Adapted from https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/engine/trainer.py
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-import logging
import torch
import os
from bob.ip.binseg.utils.model_serialization import load_state_dict
from bob.ip.binseg.utils.model_zoo import cache_url
+import logging
+logger = logging.getLogger(__name__)
+
+
class Checkpointer:
- """Adapted from `maskrcnn-benchmark`_ under MIT license
-
- Returns
- -------
- [type]
- [description]
+ """Adapted from `maskrcnn-benchmark
+ <https://github.com/facebookresearch/maskrcnn-benchmark>`_ under MIT license
"""
+
def __init__(
self,
model,
@@ -25,16 +24,12 @@ class Checkpointer:
scheduler=None,
save_dir="",
save_to_disk=None,
- logger=None,
):
self.model = model
self.optimizer = optimizer
self.scheduler = scheduler
self.save_dir = save_dir
self.save_to_disk = save_to_disk
- if logger is None:
- logger = logging.getLogger(__name__)
- self.logger = logger
def save(self, name, **kwargs):
if not self.save_dir:
@@ -51,10 +46,11 @@ class Checkpointer:
data["scheduler"] = self.scheduler.state_dict()
data.update(kwargs)
- save_file = os.path.join(self.save_dir, "{}.pth".format(name))
- self.logger.info("Saving checkpoint to {}".format(save_file))
+ dest_filename = f"{name}.pth"
+ save_file = os.path.join(self.save_dir, dest_filename)
+ logger.info(f"Saving checkpoint to {save_file}")
torch.save(data, save_file)
- self.tag_last_checkpoint(save_file)
+ self.tag_last_checkpoint(dest_filename)
def load(self, f=None):
if self.has_checkpoint():
@@ -62,16 +58,16 @@ class Checkpointer:
f = self.get_checkpoint_file()
if not f:
# no checkpoint could be found
- self.logger.warn("No checkpoint found. Initializing model from scratch")
+ logger.warn("No checkpoint found. Initializing model from scratch")
return {}
- self.logger.info("Loading checkpoint from {}".format(f))
checkpoint = self._load_file(f)
self._load_model(checkpoint)
+ actual_file = os.path.join(self.save_dir, f)
if "optimizer" in checkpoint and self.optimizer:
- self.logger.info("Loading optimizer from {}".format(f))
+ logger.info(f"Loading optimizer from {actual_file}")
self.optimizer.load_state_dict(checkpoint.pop("optimizer"))
if "scheduler" in checkpoint and self.scheduler:
- self.logger.info("Loading scheduler from {}".format(f))
+ logger.info(f"Loading scheduler from {actual_file}")
self.scheduler.load_state_dict(checkpoint.pop("scheduler"))
# return any further checkpoint data
@@ -99,7 +95,9 @@ class Checkpointer:
f.write(last_filename)
def _load_file(self, f):
- return torch.load(f, map_location=torch.device("cpu"))
+ actual_file = os.path.join(self.save_dir, f)
+ logger.info(f"Loading checkpoint from {actual_file}")
+ return torch.load(actual_file, map_location=torch.device("cpu"))
def _load_model(self, checkpoint):
load_state_dict(self.model, checkpoint.pop("model"))
@@ -113,10 +111,9 @@ class DetectronCheckpointer(Checkpointer):
scheduler=None,
save_dir="",
save_to_disk=None,
- logger=None,
):
super(DetectronCheckpointer, self).__init__(
- model, optimizer, scheduler, save_dir, save_to_disk, logger
+ model, optimizer, scheduler, save_dir, save_to_disk
)
def _load_file(self, f):
@@ -124,10 +121,10 @@ class DetectronCheckpointer(Checkpointer):
if f.startswith("http"):
# if the file is a url path, download it and cache it
cached_f = cache_url(f)
- self.logger.info("url {} cached in {}".format(f, cached_f))
+ logger.info(f"url {f} cached in {cached_f}")
f = cached_f
# load checkpoint
loaded = super(DetectronCheckpointer, self)._load_file(f)
if "model" not in loaded:
loaded = dict(model=loaded)
- return loaded
\ No newline at end of file
+ return loaded
diff --git a/bob/ip/binseg/utils/click.py b/bob/ip/binseg/utils/click.py
deleted file mode 100644
index 8b8294d97f869167f0908f22c4521c6fcda1a243..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/utils/click.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-
-import click
-
-class OptionEatAll(click.Option):
- """
- Allows for *args and **kwargs to be passed to click
- https://stackoverflow.com/questions/48391777/nargs-equivalent-for-options-in-click
- """
-
- def __init__(self, *args, **kwargs):
- self.save_other_options = kwargs.pop('save_other_options', True)
- nargs = kwargs.pop('nargs', -1)
- assert nargs == -1, 'nargs, if set, must be -1 not {}'.format(nargs)
- super(OptionEatAll, self).__init__(*args, **kwargs)
- self._previous_parser_process = None
- self._eat_all_parser = None
-
- def add_to_parser(self, parser, ctx):
-
- def parser_process(value, state):
- # method to hook to the parser.process
- done = False
- value = [value]
- if self.save_other_options:
- # grab everything up to the next option
- while state.rargs and not done:
- for prefix in self._eat_all_parser.prefixes:
- if state.rargs[0].startswith(prefix):
- done = True
- if not done:
- value.append(state.rargs.pop(0))
- else:
- # grab everything remaining
- value += state.rargs
- state.rargs[:] = []
- value = tuple(value)
-
- # call the actual process
- self._previous_parser_process(value, state)
-
- retval = super(OptionEatAll, self).add_to_parser(parser, ctx)
- for name in self.opts:
- our_parser = parser._long_opt.get(name) or parser._short_opt.get(name)
- if our_parser:
- self._eat_all_parser = our_parser
- self._previous_parser_process = our_parser.process
- our_parser.process = parser_process
- break
- return retval
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/evaluate.py b/bob/ip/binseg/utils/evaluate.py
deleted file mode 100644
index 99259f412be8f00e7b2ea2572ba654689a4cf92d..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/utils/evaluate.py
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# only use to evaluate 2nd human annotator
-#
-import os
-import logging
-import time
-import datetime
-import numpy as np
-import torch
-import pandas as pd
-import torchvision.transforms.functional as VF
-from tqdm import tqdm
-
-from bob.ip.binseg.utils.metric import SmoothedValue, base_metrics
-from bob.ip.binseg.utils.plot import precision_recall_f1iso, precision_recall_f1iso_confintval
-from bob.ip.binseg.utils.summary import summary
-from PIL import Image
-from torchvision.transforms.functional import to_tensor
-
-
-def batch_metrics(predictions, ground_truths, names, output_folder, logger):
- """
- Calculates metrics on the batch and saves it to disc
-
- Parameters
- ----------
- predictions : :py:class:`torch.Tensor`
- tensor with pixel-wise probabilities
- ground_truths : :py:class:`torch.Tensor`
- tensor with binary ground-truth
- names : list
- list of file names
- output_folder : str
- output path
- logger : :py:class:`logging.Logger`
- python logger
-
- Returns
- -------
- list
- list containing batch metrics: ``[name, threshold, precision, recall, specificity, accuracy, jaccard, f1_score]``
- """
- step_size = 0.01
- batch_metrics = []
-
- for j in range(predictions.size()[0]):
- # ground truth byte
- gts = ground_truths[j].byte()
-
- file_name = "{}.csv".format(names[j])
- logger.info("saving {}".format(file_name))
-
- with open (os.path.join(output_folder,file_name), "w+") as outfile:
-
- outfile.write("threshold, precision, recall, specificity, accuracy, jaccard, f1_score\n")
-
- for threshold in np.arange(0.0,1.0,step_size):
- # threshold
- binary_pred = torch.gt(predictions[j], threshold).byte()
-
- # equals and not-equals
- equals = torch.eq(binary_pred, gts) # tensor
- notequals = torch.ne(binary_pred, gts) # tensor
-
- # true positives
- tp_tensor = (gts * binary_pred ) # tensor
- tp_count = torch.sum(tp_tensor).item() # scalar
-
- # false positives
- fp_tensor = torch.eq((binary_pred + tp_tensor), 1)
- fp_count = torch.sum(fp_tensor).item()
-
- # true negatives
- tn_tensor = equals - tp_tensor
- tn_count = torch.sum(tn_tensor).item()
-
- # false negatives
- fn_tensor = notequals - fp_tensor
- fn_count = torch.sum(fn_tensor).item()
-
- # calc metrics
- metrics = base_metrics(tp_count, fp_count, tn_count, fn_count)
-
- # write to disk
- outfile.write("{:.2f},{:.5f},{:.5f},{:.5f},{:.5f},{:.5f},{:.5f} \n".format(threshold, *metrics))
-
- batch_metrics.append([names[j],threshold, *metrics ])
-
-
- return batch_metrics
-
-
-
-def do_eval(
- prediction_folder,
- data_loader,
- output_folder = None,
- title = '2nd human',
- legend = '2nd human',
- prediction_extension = None,
-):
-
- """
- Calculate metrics on saved prediction images (needs batch_size = 1 !)
-
- Parameters
- ---------
- model : :py:class:`torch.nn.Module`
- neural network model (e.g. DRIU, HED, UNet)
- data_loader : py:class:`torch.torch.utils.data.DataLoader`
- device : str
- device to use ``'cpu'`` or ``'cuda'``
- output_folder : str
- """
- logger = logging.getLogger("bob.ip.binseg.engine.evaluate")
- logger.info("Start evaluation")
- logger.info("Prediction folder {}".format(prediction_folder))
- results_subfolder = os.path.join(output_folder,'results')
- os.makedirs(results_subfolder,exist_ok=True)
-
-
- # Collect overall metrics
- metrics = []
- num_images = len(data_loader)
- for samples in tqdm(data_loader):
- names = samples[0]
- images = samples[1]
- ground_truths = samples[2]
-
- if prediction_extension is None:
- pred_file = os.path.join(prediction_folder,names[0])
- else:
- pred_file = os.path.join(prediction_folder,os.path.splitext(names[0])[0] + '.png')
- probabilities = Image.open(pred_file)
- probabilities = probabilities.convert(mode='L')
- probabilities = to_tensor(probabilities)
-
-
- b_metrics = batch_metrics(probabilities, ground_truths, names,results_subfolder, logger)
- metrics.extend(b_metrics)
-
-
-
- # DataFrame
- df_metrics = pd.DataFrame(metrics,columns= \
- ["name",
- "threshold",
- "precision",
- "recall",
- "specificity",
- "accuracy",
- "jaccard",
- "f1_score"])
-
- # Report and Averages
- metrics_file = "Metrics.csv"
- metrics_path = os.path.join(results_subfolder, metrics_file)
- logger.info("Saving average over all input images: {}".format(metrics_file))
-
- avg_metrics = df_metrics.groupby('threshold').mean()
- std_metrics = df_metrics.groupby('threshold').std()
-
- # Uncomment below for F1-score calculation based on average precision and metrics instead of
- # F1-scores of individual images. This method is in line with Maninis et. al. (2016)
- #avg_metrics["f1_score"] = (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
- # (avg_metrics["precision"]+avg_metrics["recall"])
-
-
- avg_metrics["std_pr"] = std_metrics["precision"]
- avg_metrics["pr_upper"] = avg_metrics['precision'] + avg_metrics["std_pr"]
- avg_metrics["pr_lower"] = avg_metrics['precision'] - avg_metrics["std_pr"]
- avg_metrics["std_re"] = std_metrics["recall"]
- avg_metrics["re_upper"] = avg_metrics['recall'] + avg_metrics["std_re"]
- avg_metrics["re_lower"] = avg_metrics['recall'] - avg_metrics["std_re"]
- avg_metrics["std_f1"] = std_metrics["f1_score"]
-
- avg_metrics.to_csv(metrics_path)
- maxf1 = avg_metrics['f1_score'].max()
- optimal_f1_threshold = avg_metrics['f1_score'].idxmax()
-
- logger.info("Highest F1-score of {:.5f}, achieved at threshold {}".format(maxf1, optimal_f1_threshold))
-
- # Plotting
- #print(avg_metrics)
- np_avg_metrics = avg_metrics.to_numpy().T
- fig_name = "precision_recall.pdf"
- logger.info("saving {}".format(fig_name))
- fig = precision_recall_f1iso_confintval([np_avg_metrics[0]],[np_avg_metrics[1]],[np_avg_metrics[7]],[np_avg_metrics[8]],[np_avg_metrics[10]],[np_avg_metrics[11]], [legend ,None], title=title)
- fig_filename = os.path.join(results_subfolder, fig_name)
- fig.savefig(fig_filename)
-
-
-
diff --git a/bob/ip/binseg/utils/metric.py b/bob/ip/binseg/utils/metric.py
index bcb91511f533a8ed69843487ef8cbe793e42a925..903836f6ef17b231ecb942efc4156c3408cff885 100644
--- a/bob/ip/binseg/utils/metric.py
+++ b/bob/ip/binseg/utils/metric.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-from collections import defaultdict
from collections import deque
import torch
@@ -27,10 +26,11 @@ class SmoothedValue:
d = torch.tensor(list(self.deque))
return d.mean().item()
+
def base_metrics(tp, fp, tn, fn):
"""
Calculates Precision, Recall (=Sensitivity), Specificity, Accuracy, Jaccard and F1-score (Dice)
-
+
Parameters
----------
@@ -39,7 +39,7 @@ def base_metrics(tp, fp, tn, fn):
True positives
fp : float
- False positives
+ False positives
tn : float
True negatives
@@ -52,13 +52,36 @@ def base_metrics(tp, fp, tn, fn):
-------
metrics : list
-
+
"""
- precision = tp / (tp + fp + ( (tp+fp) == 0) )
- recall = tp / (tp + fn + ( (tp+fn) == 0) )
- specificity = tn / (fp + tn + ( (fp+tn) == 0) )
- accuracy = (tp + tn) / (tp+fp+fn+tn)
- jaccard = tp / (tp+fp+fn + ( (tp+fp+fn) == 0) )
- f1_score = (2.0 * tp ) / (2.0 * tp + fp + fn + ( (2.0 * tp + fp + fn) == 0) )
- #f1_score = (2.0 * precision * recall) / (precision + recall)
- return [precision, recall, specificity, accuracy, jaccard, f1_score]
\ No newline at end of file
+ precision = tp / (tp + fp + ((tp + fp) == 0))
+ recall = tp / (tp + fn + ((tp + fn) == 0))
+ specificity = tn / (fp + tn + ((fp + tn) == 0))
+ accuracy = (tp + tn) / (tp + fp + fn + tn)
+ jaccard = tp / (tp + fp + fn + ((tp + fp + fn) == 0))
+ f1_score = (2.0 * tp) / (2.0 * tp + fp + fn + ((2.0 * tp + fp + fn) == 0))
+ # f1_score = (2.0 * precision * recall) / (precision + recall)
+ return [precision, recall, specificity, accuracy, jaccard, f1_score]
+
+
+def auc(precision, recall):
+ """Calculates the area under the precision-recall curve (AUC)
+
+ .. todo:: Integrate this to metrics reporting in compare.py
+ """
+
+ rec_unique, rec_unique_ndx = numpy.unique(recall, return_index=True)
+
+ prec_unique = precision[rec_unique_ndx]
+
+ if rec_unique.shape[0] > 1:
+ prec_interp = numpy.interp(
+ numpy.arange(0, 1, 0.01),
+ rec_unique,
+ prec_unique,
+ left=0.0,
+ right=0.0,
+ )
+ return prec_interp.sum() * 0.01
+
+ return 0.0
diff --git a/bob/ip/binseg/utils/model_serialization.py b/bob/ip/binseg/utils/model_serialization.py
index 84ff2491ea85751cc0b0910d278f035e9571f0eb..4c84e84f7e57364c128cc88c0116c8aec5e2bcc5 100644
--- a/bob/ip/binseg/utils/model_serialization.py
+++ b/bob/ip/binseg/utils/model_serialization.py
@@ -1,10 +1,14 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# https://github.com/facebookresearch/maskrcnn-benchmark
+
from collections import OrderedDict
+
import logging
+logger = logging.getLogger(__name__)
import torch
+
def align_and_update_state_dicts(model_state_dict, loaded_state_dict):
"""
Strategy: suppose that the models that we will create will have prefixes appended
@@ -38,14 +42,13 @@ def align_and_update_state_dicts(model_state_dict, loaded_state_dict):
max_size = max([len(key) for key in current_keys]) if current_keys else 1
max_size_loaded = max([len(key) for key in loaded_keys]) if loaded_keys else 1
log_str_template = "{: <{}} loaded from {: <{}} of shape {}"
- logger = logging.getLogger(__name__)
for idx_new, idx_old in enumerate(idxs.tolist()):
if idx_old == -1:
continue
key = current_keys[idx_new]
key_old = loaded_keys[idx_old]
model_state_dict[key] = loaded_state_dict[key_old]
- logger.info(
+ logger.debug(
log_str_template.format(
key,
max_size,
@@ -75,4 +78,4 @@ def load_state_dict(model, loaded_state_dict):
align_and_update_state_dicts(model_state_dict, loaded_state_dict)
# use strict loading
- model.load_state_dict(model_state_dict)
\ No newline at end of file
+ model.load_state_dict(model_state_dict)
diff --git a/bob/ip/binseg/utils/model_zoo.py b/bob/ip/binseg/utils/model_zoo.py
index 8bc7c931e0e05d682e03cc52c22413c2f185965e..2eb98f552f1654043b5bf4a03ae4e4627a65c944 100644
--- a/bob/ip/binseg/utils/model_zoo.py
+++ b/bob/ip/binseg/utils/model_zoo.py
@@ -5,19 +5,15 @@
# https://github.com/pytorch/pytorch/blob/master/torch/hub.py
# https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/utils/checkpoint.py
-import errno
import hashlib
import os
import re
import shutil
import sys
import tempfile
-import torch
-import warnings
-import zipfile
from urllib.request import urlopen
from urllib.parse import urlparse
-from tqdm import tqdm
+from tqdm import tqdm
modelurls = {
"vgg11": "https://download.pytorch.org/models/vgg11-bbd30ac9.pth",
@@ -33,15 +29,19 @@ modelurls = {
"resnet50": "https://download.pytorch.org/models/resnet50-19c8e357.pth",
"resnet101": "https://download.pytorch.org/models/resnet101-5d3b4d8f.pth",
"resnet152": "https://download.pytorch.org/models/resnet152-b121ed2d.pth",
- "resnet50_SIN_IN": "https://bitbucket.org/robert_geirhos/texture-vs-shape-pretrained-models/raw/60b770e128fffcbd8562a3ab3546c1a735432d03/resnet50_finetune_60_epochs_lr_decay_after_30_start_resnet50_train_45_epochs_combined_IN_SF-ca06340c.pth.tar",
- "mobilenetv2": "https://dl.dropboxusercontent.com/s/4nie4ygivq04p8y/mobilenet_v2.pth.tar",
- }
+ #"resnet50_SIN_IN": "https://bitbucket.org/robert_geirhos/texture-vs-shape-pretrained-models/raw/60b770e128fffcbd8562a3ab3546c1a735432d03/resnet50_finetune_60_epochs_lr_decay_after_30_start_resnet50_train_45_epochs_combined_IN_SF-ca06340c.pth.tar",
+ "resnet50_SIN_IN": "http://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/resnet50_finetune_60_epochs_lr_decay_after_30_start_resnet50_train_45_epochs_combined_IN_SF-ca06340c.pth.tar",
+ #"mobilenetv2": "https://dl.dropboxusercontent.com/s/4nie4ygivq04p8y/mobilenet_v2.pth.tar",
+ "mobilenetv2": "http://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/mobilenet_v2.pth.tar",
+}
+"""URLs of pre-trained models (backbones)"""
-def _download_url_to_file(url, dst, hash_prefix, progress):
+
+def download_url_to_file(url, dst, hash_prefix, progress):
file_size = None
u = urlopen(url)
meta = u.info()
- if hasattr(meta, 'getheaders'):
+ if hasattr(meta, "getheaders"):
content_length = meta.getheaders("Content-Length")
else:
content_length = meta.get_all("Content-Length")
@@ -65,16 +65,21 @@ def _download_url_to_file(url, dst, hash_prefix, progress):
f.close()
if hash_prefix is not None:
digest = sha256.hexdigest()
- if digest[:len(hash_prefix)] != hash_prefix:
- raise RuntimeError('invalid hash value (expected "{}", got "{}")'
- .format(hash_prefix, digest))
+ if digest[: len(hash_prefix)] != hash_prefix:
+ raise RuntimeError(
+ 'invalid hash value (expected "{}", got "{}")'.format(
+ hash_prefix, digest
+ )
+ )
shutil.move(f.name, dst)
finally:
f.close()
if os.path.exists(f.name):
os.remove(f.name)
-HASH_REGEX = re.compile(r'-([a-f0-9]*)\.')
+
+HASH_REGEX = re.compile(r"-([a-f0-9]*)\.")
+
def cache_url(url, model_dir=None, progress=True):
r"""Loads the Torch serialized object at the given URL.
@@ -99,13 +104,13 @@ def cache_url(url, model_dir=None, progress=True):
os.makedirs(model_dir)
parts = urlparse(url)
filename = os.path.basename(parts.path)
-
+
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename)
if hash_prefix is not None:
hash_prefix = hash_prefix.group(1)
- _download_url_to_file(url, cached_file, hash_prefix, progress=progress)
-
- return cached_file
\ No newline at end of file
+ download_url_to_file(url, cached_file, hash_prefix, progress=progress)
+
+ return cached_file
diff --git a/bob/ip/binseg/utils/plot.py b/bob/ip/binseg/utils/plot.py
index de1d531d8bdab9db1f2263f4221157a3cca24912..9be1b0edec5f4ddd298df6561a48ac1009b3deab 100644
--- a/bob/ip/binseg/utils/plot.py
+++ b/bob/ip/binseg/utils/plot.py
@@ -1,454 +1,271 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import numpy as np
-import os
-import csv
-import pandas as pd
-import PIL
-from PIL import Image,ImageFont, ImageDraw
-import torchvision.transforms.functional as VF
-import torch
-
-def precision_recall_f1iso(precision, recall, names, title=None):
- """
- Author: Andre Anjos (andre.anjos@idiap.ch).
+import contextlib
+from itertools import cycle
- Creates a precision-recall plot of the given data.
- The plot will be annotated with F1-score iso-lines (in which the F1-score
- maintains the same value)
+import numpy
+import pandas
- Parameters
- ----------
- precision : :py:class:`numpy.ndarray` or :py:class:`list`
- A list of 1D np arrays containing the Y coordinates of the plot, or
- the precision, or a 2D np array in which the rows correspond to each
- of the system's precision coordinates.
- recall : :py:class:`numpy.ndarray` or :py:class:`list`
- A list of 1D np arrays containing the X coordinates of the plot, or
- the recall, or a 2D np array in which the rows correspond to each
- of the system's recall coordinates.
- names : :py:class:`list`
- An iterable over the names of each of the systems along the rows of
- ``precision`` and ``recall``
- title : :py:class:`str`, optional
- A title for the plot. If not set, omits the title
+import matplotlib
+matplotlib.use("agg")
- Returns
- -------
- matplotlib.figure.Figure
- A matplotlib figure you can save or display
- """
- import matplotlib
- matplotlib.use('agg')
- import matplotlib.pyplot as plt
- from itertools import cycle
- fig, ax1 = plt.subplots(1)
- lines = ["-","--","-.",":"]
- linecycler = cycle(lines)
- for p, r, n in zip(precision, recall, names):
- # Plots only from the point where recall reaches its maximum, otherwise, we
- # don't see a curve...
- i = r.argmax()
- pi = p[i:]
- ri = r[i:]
- valid = (pi+ri) > 0
- f1 = 2 * (pi[valid]*ri[valid]) / (pi[valid]+ri[valid])
- # optimal point along the curve
- argmax = f1.argmax()
- opi = pi[argmax]
- ori = ri[argmax]
- # Plot Recall/Precision as threshold changes
- ax1.plot(ri[pi>0], pi[pi>0], next(linecycler), label='[F={:.4f}] {}'.format(f1.max(), n),)
- ax1.plot(ori,opi, marker='o', linestyle=None, markersize=3, color='black')
- ax1.grid(linestyle='--', linewidth=1, color='gray', alpha=0.2)
- if len(names) > 1:
- plt.legend(loc='lower left', framealpha=0.5)
- ax1.set_xlabel('Recall')
- ax1.set_ylabel('Precision')
- ax1.set_xlim([0.0, 1.0])
- ax1.set_ylim([0.0, 1.0])
- if title is not None: ax1.set_title(title)
- # Annotates plot with F1-score iso-lines
- ax2 = ax1.twinx()
- f_scores = np.linspace(0.1, 0.9, num=9)
- tick_locs = []
- tick_labels = []
- for f_score in f_scores:
- x = np.linspace(0.01, 1)
- y = f_score * x / (2 * x - f_score)
- l, = plt.plot(x[y >= 0], y[y >= 0], color='green', alpha=0.1)
- tick_locs.append(y[-1])
- tick_labels.append('%.1f' % f_score)
- ax2.tick_params(axis='y', which='both', pad=0, right=False, left=False)
- ax2.set_ylabel('iso-F', color='green', alpha=0.3)
- ax2.set_ylim([0.0, 1.0])
- ax2.yaxis.set_label_coords(1.015, 0.97)
- ax2.set_yticks(tick_locs) #notice these are invisible
- for k in ax2.set_yticklabels(tick_labels):
- k.set_color('green')
- k.set_alpha(0.3)
- k.set_size(8)
- # we should see some of axes 1 axes
- ax1.spines['right'].set_visible(False)
- ax1.spines['top'].set_visible(False)
- ax1.spines['left'].set_position(('data', -0.015))
- ax1.spines['bottom'].set_position(('data', -0.015))
- # we shouldn't see any of axes 2 axes
- ax2.spines['right'].set_visible(False)
- ax2.spines['top'].set_visible(False)
- ax2.spines['left'].set_visible(False)
- ax2.spines['bottom'].set_visible(False)
- plt.tight_layout()
- return fig
+import matplotlib.pyplot as plt
+
+import logging
+logger = logging.getLogger(__name__)
-def precision_recall_f1iso_confintval(precision, recall, pr_upper, pr_lower, re_upper, re_lower, names, title=None):
- """
- Author: Andre Anjos (andre.anjos@idiap.ch).
- Creates a precision-recall plot of the given data.
- The plot will be annotated with F1-score iso-lines (in which the F1-score
- maintains the same value)
+@contextlib.contextmanager
+def _precision_recall_canvas(title=None):
+ """Generates a canvas to draw precision-recall curves
+
+ Works like a context manager, yielding a figure and an axes set in which
+ the precision-recall curves should be added to. The figure already
+ contains F1-ISO lines and is preset to a 0-1 square region. Once the
+ context is finished, ``fig.tight_layout()`` is called.
+
Parameters
----------
- precision : :py:class:`numpy.ndarray` or :py:class:`list`
- A list of 1D np arrays containing the Y coordinates of the plot, or
- the precision, or a 2D np array in which the rows correspond to each
- of the system's precision coordinates.
- recall : :py:class:`numpy.ndarray` or :py:class:`list`
- A list of 1D np arrays containing the X coordinates of the plot, or
- the recall, or a 2D np array in which the rows correspond to each
- of the system's recall coordinates.
- names : :py:class:`list`
- An iterable over the names of each of the systems along the rows of
- ``precision`` and ``recall``
- title : :py:class:`str`, optional
- A title for the plot. If not set, omits the title
- Returns
- -------
- matplotlib.figure.Figure
- A matplotlib figure you can save or display
+ title : :py:class:`str`, Optional
+ Optional title to add to this plot
+
+
+ Yields
+ ------
+
+ figure : matplotlib.figure.Figure
+ The figure that should be finally returned to the user
+
+ axes : matplotlib.figure.Axes
+ An axis set where to precision-recall plots should be added to
+
"""
- import matplotlib
- matplotlib.use('agg')
- import matplotlib.pyplot as plt
- from itertools import cycle
- fig, ax1 = plt.subplots(1)
- lines = ["-","--","-.",":"]
- colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728',
- '#9467bd', '#8c564b', '#e377c2', '#7f7f7f',
- '#bcbd22', '#17becf']
- colorcycler = cycle(colors)
- linecycler = cycle(lines)
- for p, r, pu, pl, ru, rl, n in zip(precision, recall, pr_upper, pr_lower, re_upper, re_lower, names):
- # Plots only from the point where recall reaches its maximum, otherwise, we
- # don't see a curve...
- i = r.argmax()
- pi = p[i:]
- ri = r[i:]
- pui = pu[i:]
- pli = pl[i:]
- rui = ru[i:]
- rli = rl[i:]
- valid = (pi+ri) > 0
- f1 = 2 * (pi[valid]*ri[valid]) / (pi[valid]+ri[valid])
- # optimal point along the curve
- argmax = f1.argmax()
- opi = pi[argmax]
- ori = ri[argmax]
- # Plot Recall/Precision as threshold changes
- ax1.plot(ri[pi>0], pi[pi>0], next(linecycler), label='[F={:.4f}] {}'.format(f1.max(), n),)
- ax1.plot(ori,opi, marker='o', linestyle=None, markersize=3, color='black')
- # Plot confidence
- # Upper bound
- #ax1.plot(r95ui[p95ui>0], p95ui[p95ui>0])
- # Lower bound
- #ax1.plot(r95li[p95li>0], p95li[p95li>0])
- # create the limiting polygon
- vert_x = np.concatenate((rui[pui>0], rli[pli>0][::-1]))
- vert_y = np.concatenate((pui[pui>0], pli[pli>0][::-1]))
- # hacky workaround to plot 2nd human
- if np.isclose(np.mean(rui), rui[1], rtol=1e-05):
- print('found human')
- p = plt.Polygon(np.column_stack((vert_x, vert_y)), facecolor='none', alpha=.2, edgecolor=next(colorcycler),lw=2)
- else:
- p = plt.Polygon(np.column_stack((vert_x, vert_y)), facecolor=next(colorcycler), alpha=.2, edgecolor='none',lw=.2)
- ax1.add_artist(p)
-
- ax1.grid(linestyle='--', linewidth=1, color='gray', alpha=0.2)
- if len(names) > 1:
- plt.legend(loc='lower left', framealpha=0.5)
- ax1.set_xlabel('Recall')
- ax1.set_ylabel('Precision')
- ax1.set_xlim([0.0, 1.0])
- ax1.set_ylim([0.0, 1.0])
- if title is not None: ax1.set_title(title)
+
+ fig, axes1 = plt.subplots(1)
+
+ # Names and bounds
+ axes1.set_xlabel("Recall")
+ axes1.set_ylabel("Precision")
+ axes1.set_xlim([0.0, 1.0])
+ axes1.set_ylim([0.0, 1.0])
+
+ if title is not None:
+ axes1.set_title(title)
+
+ axes1.grid(linestyle="--", linewidth=1, color="gray", alpha=0.2)
+ axes2 = axes1.twinx()
+
# Annotates plot with F1-score iso-lines
- ax2 = ax1.twinx()
- f_scores = np.linspace(0.1, 0.9, num=9)
+ f_scores = numpy.linspace(0.1, 0.9, num=9)
tick_locs = []
tick_labels = []
for f_score in f_scores:
- x = np.linspace(0.01, 1)
+ x = numpy.linspace(0.01, 1)
y = f_score * x / (2 * x - f_score)
- l, = plt.plot(x[y >= 0], y[y >= 0], color='green', alpha=0.1)
+ (l,) = plt.plot(x[y >= 0], y[y >= 0], color="green", alpha=0.1)
tick_locs.append(y[-1])
- tick_labels.append('%.1f' % f_score)
- ax2.tick_params(axis='y', which='both', pad=0, right=False, left=False)
- ax2.set_ylabel('iso-F', color='green', alpha=0.3)
- ax2.set_ylim([0.0, 1.0])
- ax2.yaxis.set_label_coords(1.015, 0.97)
- ax2.set_yticks(tick_locs) #notice these are invisible
- for k in ax2.set_yticklabels(tick_labels):
- k.set_color('green')
+ tick_labels.append("%.1f" % f_score)
+ axes2.tick_params(axis="y", which="both", pad=0, right=False, left=False)
+ axes2.set_ylabel("iso-F", color="green", alpha=0.3)
+ axes2.set_ylim([0.0, 1.0])
+ axes2.yaxis.set_label_coords(1.015, 0.97)
+ axes2.set_yticks(tick_locs) # notice these are invisible
+ for k in axes2.set_yticklabels(tick_labels):
+ k.set_color("green")
k.set_alpha(0.3)
k.set_size(8)
+
# we should see some of axes 1 axes
- ax1.spines['right'].set_visible(False)
- ax1.spines['top'].set_visible(False)
- ax1.spines['left'].set_position(('data', -0.015))
- ax1.spines['bottom'].set_position(('data', -0.015))
+ axes1.spines["right"].set_visible(False)
+ axes1.spines["top"].set_visible(False)
+ axes1.spines["left"].set_position(("data", -0.015))
+ axes1.spines["bottom"].set_position(("data", -0.015))
+
# we shouldn't see any of axes 2 axes
- ax2.spines['right'].set_visible(False)
- ax2.spines['top'].set_visible(False)
- ax2.spines['left'].set_visible(False)
- ax2.spines['bottom'].set_visible(False)
+ axes2.spines["right"].set_visible(False)
+ axes2.spines["top"].set_visible(False)
+ axes2.spines["left"].set_visible(False)
+ axes2.spines["bottom"].set_visible(False)
+
+ # yield execution, lets user draw precision-recall plots, and the legend
+ # before tighteneing the layout
+ yield fig, axes1
+
plt.tight_layout()
- return fig
-def loss_curve(df, title):
- """ Creates a loss curve given a Dataframe with column names:
- ``['avg. loss', 'median loss','lr','max memory']``
+def precision_recall_f1iso(data, confidence=True):
+ """Creates a precision-recall plot with confidence intervals
+
+ This function creates and returns a Matplotlib figure with a
+ precision-recall plot containing shaded confidence intervals (standard
+ deviation on the precision-recall measurements). The plot will be
+ annotated with F1-score iso-lines (in which the F1-score maintains the same
+ value).
+
+ This function specially supports "second-annotator" entries by plotting a
+ line showing the comparison between the default annotator being analyzed
+ and a second "opinion". Second annotator dataframes contain a single
+ entry (threshold=0.5), given the nature of the binary map comparisons.
+
Parameters
----------
- df : :py:class:`pandas.DataFrame`
- Returns
- -------
- matplotlib.figure.Figure
- """
- import matplotlib
- matplotlib.use('agg')
- import matplotlib.pyplot as plt
- ax1 = df.plot(y="median loss", grid=True)
- ax1.set_title(title)
- ax1.set_ylabel('median loss')
- ax1.grid(linestyle='--', linewidth=1, color='gray', alpha=0.2)
- ax2 = df['lr'].plot(secondary_y=True,legend=True,grid=True,)
- ax2.set_ylabel('lr')
- ax1.set_xlabel('epoch')
- plt.tight_layout()
- fig = ax1.get_figure()
- return fig
+ data : dict
+ A dictionary in which keys are strings defining plot labels and values
+ are dictionaries with two entries:
+ * ``df``: :py:class:`pandas.DataFrame`
-def read_metricscsv(file):
- """
- Read precision and recall from csv file
+ A dataframe that is produced by our evaluator engine, indexed by
+ integer "thresholds", containing the following columns: ``threshold``
+ (sorted ascending), ``precision``, ``recall``, ``pr_upper`` (upper
+ precision bounds), ``pr_lower`` (lower precision bounds),
+ ``re_upper`` (upper recall bounds), ``re_lower`` (lower recall
+ bounds).
- Parameters
- ----------
- file : str
- path to file
+ * ``threshold``: :py:class:`list`
+
+ A threshold to graph with a dot for each set. Specific
+ threshold values do not affect "second-annotator" dataframes.
+
+ confidence : :py:class:`bool`, Optional
+ If set, draw confidence intervals for each line, using ``*_upper`` and
+ ``*_lower`` entries.
- Returns
- -------
- :py:class:`numpy.ndarray`
- :py:class:`numpy.ndarray`
- """
- with open (file, "r") as infile:
- metricsreader = csv.reader(infile)
- # skip header row
- next(metricsreader)
- precision = []
- recall = []
- pr_upper = []
- pr_lower = []
- re_upper = []
- re_lower = []
- for row in metricsreader:
- precision.append(float(row[1]))
- recall.append(float(row[2]))
- pr_upper.append(float(row[8]))
- pr_lower.append(float(row[9]))
- re_upper.append(float(row[11]))
- re_lower.append(float(row[12]))
- return np.array(precision), np.array(recall), np.array(pr_upper), np.array(pr_lower), np.array(re_upper), np.array(re_lower)
-
-
-def plot_overview(outputfolders,title):
- """
- Plots comparison chart of all trained models
- Parameters
- ----------
- outputfolder : list
- list containing output paths of all evaluated models (e.g. ``['DRIVE/model1', 'DRIVE/model2']``)
- title : str
- title of plot
Returns
-------
- matplotlib.figure.Figure
+
+ figure : matplotlib.figure.Figure
+ A matplotlib figure you can save or display (uses an ``agg`` backend)
+
"""
- precisions = []
- recalls = []
- pr_ups = []
- pr_lows = []
- re_ups = []
- re_lows = []
- names = []
- params = []
- for folder in outputfolders:
- # metrics
- metrics_path = os.path.join(folder,'results/Metrics.csv')
- pr, re, pr_upper, pr_lower, re_upper, re_lower = read_metricscsv(metrics_path)
- precisions.append(pr)
- recalls.append(re)
- pr_ups.append(pr_upper)
- pr_lows.append(pr_lower)
- re_ups.append(re_upper)
- re_lows.append(re_lower)
- modelname = folder.split('/')[-1]
- name = '{} '.format(modelname)
- names.append(name)
- #title = folder.split('/')[-4]
- fig = precision_recall_f1iso_confintval(precisions,recalls, pr_ups, pr_lows, re_ups, re_lows, names,title)
+
+ lines = ["-", "--", "-.", ":"]
+ colors = [
+ "#1f77b4",
+ "#ff7f0e",
+ "#2ca02c",
+ "#d62728",
+ "#9467bd",
+ "#8c564b",
+ "#e377c2",
+ "#7f7f7f",
+ "#bcbd22",
+ "#17becf",
+ ]
+ colorcycler = cycle(colors)
+ linecycler = cycle(lines)
+
+ with _precision_recall_canvas(title=None) as (fig, axes):
+
+ legend = []
+
+ for name, value in data.items():
+
+ df = value["df"]
+ threshold = value["threshold"]
+
+ # plots only from the point where recall reaches its maximum,
+ # otherwise, we don't see a curve...
+ max_recall = df["recall"].idxmax()
+ pi = df.precision[max_recall:]
+ ri = df.recall[max_recall:]
+
+ valid = (pi + ri) > 0
+ f1 = 2 * (pi[valid] * ri[valid]) / (pi[valid] + ri[valid])
+
+ # optimal point along the curve
+ bins = len(df)
+ index = int(round(bins*threshold))
+ index = min(index, len(df)-1) #avoids out of range indexing
+
+ # plots Recall/Precision as threshold changes
+ label = f"{name} (F1={df.f1_score[index]:.4f})"
+ color = next(colorcycler)
+
+ if len(df) == 1:
+ # plot black dot for F1-score at select threshold
+ marker, = axes.plot(df.recall[index], df.precision[index],
+ marker="*", markersize=6, color=color, alpha=0.8,
+ linestyle="None")
+ line, = axes.plot(df.recall[index], df.precision[index],
+ linestyle="None", color=color, alpha=0.2)
+ legend.append(([marker, line], label))
+ else:
+ # line first, so marker gets on top
+ style = next(linecycler)
+ line, = axes.plot(ri[pi > 0], pi[pi > 0], color=color,
+ linestyle=style)
+ marker, = axes.plot(df.recall[index], df.precision[index],
+ marker="o", linestyle=style, markersize=4,
+ color=color, alpha=0.8)
+ legend.append(([marker, line], label))
+
+ if confidence:
+
+ pui = df.pr_upper[max_recall:]
+ pli = df.pr_lower[max_recall:]
+ rui = df.re_upper[max_recall:]
+ rli = df.re_lower[max_recall:]
+
+ # Plot confidence
+ # Upper bound
+ # create the limiting polygon
+ vert_x = numpy.concatenate((rui[pui > 0], rli[pli > 0][::-1]))
+ vert_y = numpy.concatenate((pui[pui > 0], pli[pli > 0][::-1]))
+
+ # hacky workaround to plot 2nd human
+ if len(df) == 1: #binary system, very likely
+ logger.warning("Found 2nd human annotator - patching...")
+ p, = axes.plot(vert_x, vert_y, color=color, alpha=0.1, lw=3)
+ else:
+ p = plt.Polygon(
+ numpy.column_stack((vert_x, vert_y)),
+ facecolor=color,
+ alpha=0.2,
+ edgecolor="none",
+ lw=0.2,
+ )
+ legend[-1][0].append(p)
+ axes.add_artist(p)
+
+ if len(label) > 1:
+ axes.legend([tuple(k[0]) for k in legend], [k[1] for k in legend],
+ loc="lower left", fancybox=True, framealpha=0.7)
+
return fig
-def metricsviz(dataset
- ,output_path
- ,tp_color= (0,255,0) # (128,128,128) Gray
- ,fp_color = (0, 0, 255) # (70, 240, 240) Cyan
- ,fn_color = (255, 0, 0) # (245, 130, 48) Orange
- ,overlayed=True):
- """ Visualizes true positives, false positives and false negatives
- Default colors TP: Gray, FP: Cyan, FN: Orange
+
+def loss_curve(df):
+ """Creates a loss curve in a Matplotlib figure.
Parameters
----------
- dataset : :py:class:`torch.utils.data.Dataset`
- output_path : str
- path where results and probability output images are stored. E.g. ``'DRIVE/MODEL'``
- tp_color : tuple
- RGB values, by default (128,128,128)
- fp_color : tuple
- RGB values, by default (70, 240, 240)
- fn_color : tuple
- RGB values, by default (245, 130, 48)
- """
- for sample in dataset:
- # get sample
- name = sample[0]
- img = VF.to_pil_image(sample[1]) # PIL Image
- gt = sample[2].byte() # byte tensor
-
- # read metrics
- metrics = pd.read_csv(os.path.join(output_path,'results','Metrics.csv'))
- optimal_threshold = metrics['threshold'][metrics['f1_score'].idxmax()]
-
- # read probability output
- pred = Image.open(os.path.join(output_path,'images',name))
- pred = pred.convert(mode='L')
- pred = VF.to_tensor(pred)
- binary_pred = torch.gt(pred, optimal_threshold).byte()
-
- # calc metrics
- # equals and not-equals
- equals = torch.eq(binary_pred, gt) # tensor
- notequals = torch.ne(binary_pred, gt) # tensor
- # true positives
- tp_tensor = (gt * binary_pred ) # tensor
- tp_pil = VF.to_pil_image(tp_tensor.float())
- tp_pil_colored = PIL.ImageOps.colorize(tp_pil, (0,0,0), tp_color)
- # false positives
- fp_tensor = torch.eq((binary_pred + tp_tensor), 1)
- fp_pil = VF.to_pil_image(fp_tensor.float())
- fp_pil_colored = PIL.ImageOps.colorize(fp_pil, (0,0,0), fp_color)
- # false negatives
- fn_tensor = notequals - fp_tensor
- fn_pil = VF.to_pil_image(fn_tensor.float())
- fn_pil_colored = PIL.ImageOps.colorize(fn_pil, (0,0,0), fn_color)
-
- # paste together
- tp_pil_colored.paste(fp_pil_colored,mask=fp_pil)
- tp_pil_colored.paste(fn_pil_colored,mask=fn_pil)
-
- if overlayed:
- tp_pil_colored = PIL.Image.blend(img, tp_pil_colored, 0.4)
- img_metrics = pd.read_csv(os.path.join(output_path,'results',name+'.csv'))
- f1 = img_metrics[' f1_score'].max()
- # add f1-score
- fnt_size = tp_pil_colored.size[1]//25
- draw = ImageDraw.Draw(tp_pil_colored)
- fnt = ImageFont.truetype('FreeMono.ttf', fnt_size)
- draw.text((0, 0),"F1: {:.4f}".format(f1),(255,255,255),font=fnt)
-
- # save to disk
- overlayed_path = os.path.join(output_path,'tpfnfpviz')
- fullpath = os.path.join(overlayed_path, name)
- fulldir = os.path.dirname(fullpath)
- if not os.path.exists(fulldir): os.makedirs(fulldir)
- tp_pil_colored.save(fullpath)
-
-
-def overlay(dataset, output_path):
- """Overlays prediction probabilities vessel tree with original test image.
+ df : :py:class:`pandas.DataFrame`
+ A dataframe containing, at least, "epoch", "median-loss" and
+ "learning-rate" columns, that will be plotted.
- Parameters
- ----------
- dataset : :py:class:`torch.utils.data.Dataset`
- output_path : str
- path where results and probability output images are stored. E.g. ``'DRIVE/MODEL'``
- """
+ Returns
+ -------
- for sample in dataset:
- # get sample
- name = sample[0]
- img = VF.to_pil_image(sample[1]) # PIL Image
-
- # read probability output
- pred = Image.open(os.path.join(output_path,'images',name)).convert(mode='L')
- # color and overlay
- pred_green = PIL.ImageOps.colorize(pred, (0,0,0), (0,255,0))
- overlayed = PIL.Image.blend(img, pred_green, 0.4)
-
- # add f1-score
- #fnt_size = overlayed.size[1]//25
- #draw = ImageDraw.Draw(overlayed)
- #fnt = ImageFont.truetype('FreeMono.ttf', fnt_size)
- #draw.text((0, 0),"F1: {:.4f}".format(f1),(255,255,255),font=fnt)
- # save to disk
- overlayed_path = os.path.join(output_path,'overlayed')
- fullpath = os.path.join(overlayed_path, name)
- fulldir = os.path.dirname(fullpath)
- if not os.path.exists(fulldir): os.makedirs(fulldir)
- overlayed.save(fullpath)
-
-
-def savetransformedtest(dataset, output_path):
- """Save the test images as they are fed into the neural network.
- Makes it easier to create overlay animations (e.g. slide)
+ figure : matplotlib.figure.Figure
+ A figure, that may be saved or displayed
- Parameters
- ----------
- dataset : :py:class:`torch.utils.data.Dataset`
- output_path : str
- path where results and probability output images are stored. E.g. ``'DRIVE/MODEL'``
"""
- for sample in dataset:
- # get sample
- name = sample[0]
- img = VF.to_pil_image(sample[1]) # PIL Image
-
- # save to disk
- testimg_path = os.path.join(output_path,'transformedtestimages')
- fullpath = os.path.join(testimg_path, name)
- fulldir = os.path.dirname(fullpath)
- if not os.path.exists(fulldir): os.makedirs(fulldir)
- img.save(fullpath)
+ ax1 = df.plot(x="epoch", y="median-loss", grid=True)
+ ax1.set_ylabel("Median Loss")
+ ax1.grid(linestyle="--", linewidth=1, color="gray", alpha=0.2)
+ ax2 = df["learning-rate"].plot(secondary_y=True, legend=True, grid=True,)
+ ax2.set_ylabel("Learning Rate")
+ ax1.set_xlabel("Epoch")
+ plt.tight_layout()
+ fig = ax1.get_figure()
+ return fig
diff --git a/bob/ip/binseg/utils/resources.py b/bob/ip/binseg/utils/resources.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea64657ca7f4f1dea3684fc0499eb269a8ffe2e4
--- /dev/null
+++ b/bob/ip/binseg/utils/resources.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Tools for interacting with the running computer or GPU"""
+
+import os
+import subprocess
+import shutil
+
+import psutil
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+_nvidia_smi = shutil.which("nvidia-smi")
+"""Location of the nvidia-smi program, if one exists"""
+
+
+GB = float(2 ** 30)
+"""The number of bytes in a gigabyte"""
+
+
+def run_nvidia_smi(query, rename=None):
+ """Returns GPU information from query
+
+ For a comprehensive list of options and help, execute ``nvidia-smi
+ --help-query-gpu`` on a host with a GPU
+
+
+ Parameters
+ ----------
+
+ query : list
+ A list of query strings as defined by ``nvidia-smi --help-query-gpu``
+
+ rename : :py:class:`list`, Optional
+ A list of keys to yield in the return value for each entry above. It
+ gives you the opportunity to rewrite some key names for convenience.
+ This list, if provided, must be of the same length as ``query``.
+
+
+ Returns
+ -------
+
+ data : :py:class:`tuple`, None
+ An ordered dictionary (organized as 2-tuples) containing the queried
+ parameters (``rename`` versions). If ``nvidia-smi`` is not available,
+ returns ``None``. Percentage information is left alone,
+ memory information is transformed to gigabytes (floating-point).
+
+ """
+
+ if _nvidia_smi is not None:
+
+ if rename is None:
+ rename = query
+ else:
+ assert len(rename) == len(query)
+
+ values = subprocess.getoutput(
+ "%s --query-gpu=%s --format=csv,noheader"
+ % (_nvidia_smi, ",".join(query))
+ )
+ values = [k.strip() for k in values.split(",")]
+ t_values = []
+ for k in values:
+ if k.endswith("%"):
+ t_values.append(float(k[:-1].strip()))
+ elif k.endswith("MiB"):
+ t_values.append(float(k[:-3].strip()) / 1024)
+ else:
+ t_values.append(k) #unchanged
+ return tuple(zip(rename, t_values))
+
+
+def gpu_constants():
+ """Returns GPU (static) information using nvidia-smi
+
+ See :py:func:`run_nvidia_smi` for operational details.
+
+ Returns
+ -------
+
+ data : :py:class:`tuple`, None
+ If ``nvidia-smi`` is not available, returns ``None``, otherwise, we
+ return an ordered dictionary (organized as 2-tuples) containing the
+ following ``nvidia-smi`` query information:
+
+ * ``gpu_name``, as ``gpu_name`` (:py:class:`str`)
+ * ``driver_version``, as ``gpu_driver_version`` (:py:class:`str`)
+ * ``memory.total``, as ``gpu_memory_total`` (transformed to gigabytes,
+ :py:class:`float`)
+
+ """
+
+ return run_nvidia_smi(
+ ("gpu_name", "driver_version", "memory.total"),
+ ("gpu_name", "gpu_driver_version", "gpu_memory_total"),
+ )
+
+
+def gpu_log():
+ """Returns GPU information about current non-static status using nvidia-smi
+
+ See :py:func:`run_nvidia_smi` for operational details.
+
+ Returns
+ -------
+
+ data : :py:class:`tuple`, None
+ If ``nvidia-smi`` is not available, returns ``None``, otherwise, we
+ return an ordered dictionary (organized as 2-tuples) containing the
+ following ``nvidia-smi`` query information:
+
+ * ``memory.used``, as ``gpu_memory_used`` (transformed to gigabytes,
+ :py:class:`float`)
+ * ``memory.free``, as ``gpu_memory_free`` (transformed to gigabytes,
+ :py:class:`float`)
+ * ``utilization.memory``, as ``gpu_memory_percent``,
+ (:py:class:`float`, in percent)
+ * ``utilization.gpu``, as ``gpu_utilization``,
+ (:py:class:`float`, in percent)
+
+ """
+
+ return run_nvidia_smi(
+ ("memory.used", "memory.free", "utilization.memory", "utilization.gpu"),
+ (
+ "gpu_memory_used",
+ "gpu_memory_free",
+ "gpu_memory_percent",
+ "gpu_percent",
+ ),
+ )
+
+
+_CLUSTER = []
+"""List of processes currently being monitored"""
+
+
+def cpu_constants():
+ """Returns static CPU information about the current system.
+
+
+ Returns
+ -------
+
+ data : tuple
+ An ordered dictionary (organized as 2-tuples) containing these entries:
+
+ 0. ``cpu_memory_total`` (:py:class:`float`): total memory available,
+ in gigabytes
+ 1. ``cpu_count`` (:py:class:`int`): number of logical CPUs available
+
+ """
+
+ return (
+ ("cpu_memory_total", psutil.virtual_memory().total / GB),
+ ("cpu_count", psutil.cpu_count(logical=True)),
+ )
+
+
+def cpu_log():
+ """Returns process (+child) information using ``psutil``.
+
+ This call examines the current process plus any spawn child and returns the
+ combined resource usage summary for the process group.
+
+
+ Returns
+ -------
+
+ data : tuple
+ An ordered dictionary (organized as 2-tuples) containing these entries:
+
+ 0. ``cpu_memory_used`` (:py:class:`float`): total memory used from
+ the system, in gigabytes
+ 1. ``cpu_rss`` (:py:class:`float`): RAM currently used by
+ process and children, in gigabytes
+ 2. ``cpu_vms`` (:py:class:`float`): total memory (RAM + swap) currently
+ used by process and children, in gigabytes
+ 3. ``cpu_percent`` (:py:class:`float`): percentage of the total CPU
+ used by this process and children (recursively) since last call
+ (first time called should be ignored). This number depends on the
+ number of CPUs in the system and can be greater than 100%
+ 4. ``cpu_processes`` (:py:class:`int`): total number of processes
+ including self and children (recursively)
+ 5. ``cpu_open_files`` (:py:class:`int`): total number of open files by
+ self and children
+
+ """
+
+ global _CLUSTER
+ if (not _CLUSTER) or (_CLUSTER[0] != psutil.Process()): # initialization
+ this = psutil.Process()
+ _CLUSTER = [this] + this.children(recursive=True)
+ # touch cpu_percent() at least once for all
+ [k.cpu_percent(interval=None) for k in _CLUSTER]
+ else:
+ # check all cluster components and update process list
+ # done so we can keep the cpu_percent() initialization
+ children = _CLUSTER[0].children()
+ stored_children = set(_CLUSTER[1:])
+ current_children = set(_CLUSTER[0].children())
+ keep_children = stored_children - current_children
+ new_children = current_children - stored_children
+ [k.cpu_percent(interval=None) for k in new_children]
+ _CLUSTER = _CLUSTER[:1] + list(keep_children) + list(new_children)
+
+ memory_info = [k.memory_info() for k in _CLUSTER]
+
+ return (
+ ("cpu_memory_used", psutil.virtual_memory().used / GB),
+ ("cpu_rss", sum([k.rss for k in memory_info]) / GB),
+ ("cpu_vms", sum([k.vms for k in memory_info]) / GB),
+ ("cpu_percent", sum(k.cpu_percent(interval=None) for k in _CLUSTER)),
+ ("cpu_processes", len(_CLUSTER)),
+ ("cpu_open_files", sum(len(k.open_files()) for k in _CLUSTER)),
+ )
diff --git a/bob/ip/binseg/utils/rsttable.py b/bob/ip/binseg/utils/rsttable.py
deleted file mode 100644
index fdc17982f8bfcdb1643baf3aff57a988352f3561..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/utils/rsttable.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import pandas as pd
-from tabulate import tabulate
-import os
-from pathlib import Path
-
-def get_paths(output_path, filename):
- """
- Parameters
- ----------
- output_path : str
- path in which to look for files
- filename : str
-
- Returns
- -------
- list
- list of file paths
- """
- datadir = Path(output_path)
- file_paths = sorted(list(datadir.glob('**/{}'.format(filename))))
- file_paths = [f.as_posix() for f in file_paths]
- return file_paths
-
-
-def create_overview_grid(output_path):
- """ Reads all Metrics.csv in a certain output path and pivots them to a rst grid table"""
- filename = 'Metrics.csv'
- metrics = get_paths(output_path,filename)
- f1s = []
- stds = []
- models = []
- databases = []
- for m in metrics:
- metrics = pd.read_csv(m)
- maxf1 = metrics['f1_score'].max()
- idmaxf1 = metrics['f1_score'].idxmax()
- std = metrics['std_f1'][idmaxf1]
- stds.append(std)
- f1s.append(maxf1)
- model = m.split('/')[-3]
- models.append(model)
- database = m.split('/')[-4]
- databases.append(database)
- df = pd.DataFrame()
- df['database'] = databases
- df['model'] = models
- df['f1'] = f1s
- df['std'] = stds
- pivot = df.pivot(index='database',columns='model',values='f1')
- pivot2 = df.pivot(index='database',columns='model',values='std')
-
- with open (os.path.join(output_path,'Metrics_overview.rst'), "w+") as outfile:
- outfile.write(tabulate(pivot,headers=pivot.columns, tablefmt="grid"))
- with open (os.path.join(output_path,'Metrics_overview_std.rst'), "w+") as outfile:
- outfile.write(tabulate(pivot2,headers=pivot2.columns, tablefmt="grid"))
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/summary.py b/bob/ip/binseg/utils/summary.py
index 127c5e66a3d7c97629ef95acebccb4f138ba43b6..7788cb7a98e0a09c09801be207232ac7ce2754e6 100644
--- a/bob/ip/binseg/utils/summary.py
+++ b/bob/ip/binseg/utils/summary.py
@@ -1,63 +1,64 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Adapted from https://github.com/pytorch/pytorch/issues/2001#issuecomment-405675488
+# Adapted from https://github.com/pytorch/pytorch/issues/2001#issuecomment-405675488
import sys
-import logging
from functools import reduce
from torch.nn.modules.module import _addindent
-from bob.ip.binseg.modeling.driu import build_driu
-def summary(model, file=sys.stderr):
- """Counts the number of paramters in each layers
-
+def summary(model):
+ """Counts the number of parameters in each model layer
+
Parameters
----------
+
model : :py:class:`torch.nn.Module`
-
+ model to summarize
+
Returns
-------
- int
+
+ repr : str
+ a multiline string representation of the network
+
+ nparam : int
number of parameters
+
"""
+
def repr(model):
+
# We treat the extra repr like the sub-module, one item per line
extra_lines = []
extra_repr = model.extra_repr()
# empty string will be split into list ['']
if extra_repr:
- extra_lines = extra_repr.split('\n')
+ extra_lines = extra_repr.split("\n")
child_lines = []
total_params = 0
for key, module in model._modules.items():
mod_str, num_params = repr(module)
mod_str = _addindent(mod_str, 2)
- child_lines.append('(' + key + '): ' + mod_str)
+ child_lines.append("(" + key + "): " + mod_str)
total_params += num_params
lines = extra_lines + child_lines
for name, p in model._parameters.items():
- if hasattr(p,'dtype'):
+ if hasattr(p, "dtype"):
total_params += reduce(lambda x, y: x * y, p.shape)
- main_str = model._get_name() + '('
+ main_str = model._get_name() + "("
if lines:
# simple one-liner info, which most builtin Modules will use
if len(extra_lines) == 1 and not child_lines:
main_str += extra_lines[0]
else:
- main_str += '\n ' + '\n '.join(lines) + '\n'
+ main_str += "\n " + "\n ".join(lines) + "\n"
- main_str += ')'
- if file is sys.stderr:
- main_str += ', \033[92m{:,}\033[0m params'.format(total_params)
- else:
- main_str += ', {:,} params'.format(total_params)
+ main_str += ")"
+ main_str += ", {:,} params".format(total_params)
return main_str, total_params
- string, count = repr(model)
- if file is not None:
- print(string, file=file)
- return count
\ No newline at end of file
+ return repr(model)
diff --git a/bob/ip/binseg/utils/table.py b/bob/ip/binseg/utils/table.py
new file mode 100644
index 0000000000000000000000000000000000000000..5feb27f8c54fd3f2a747eb649eb0690d028dfd89
--- /dev/null
+++ b/bob/ip/binseg/utils/table.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+import tabulate
+
+
+def performance_table(data, fmt):
+ """Tables result comparison in a given format
+
+
+ Parameters
+ ----------
+
+ data : dict
+ A dictionary in which keys are strings defining plot labels and values
+ are dictionaries with two entries:
+
+ * ``df``: :py:class:`pandas.DataFrame`
+
+ A dataframe that is produced by our evaluator engine, indexed by
+ integer "thresholds", containing the following columns: ``threshold``
+ (sorted ascending), ``precision``, ``recall``, ``pr_upper`` (upper
+ precision bounds), ``pr_lower`` (lower precision bounds),
+ ``re_upper`` (upper recall bounds), ``re_lower`` (lower recall
+ bounds).
+
+ * ``threshold``: :py:class:`list`
+
+ A threshold to graph with a dot for each set. Specific
+ threshold values do not affect "second-annotator" dataframes.
+
+
+ fmt : str
+ One of the formats supported by tabulate.
+
+
+ Returns
+ -------
+
+ table : str
+ A table in a specific format
+
+ """
+
+ headers = [
+ "Dataset",
+ "T",
+ "F1",
+ "F1\nstd",
+ "P",
+ "R",
+ "F1\nmax",
+ "P\nmax",
+ "R\nmax",
+ ]
+
+ table = []
+ for k, v in data.items():
+ entry = [k, v["threshold"], ]
+
+ # statistics based on the "assigned" threshold (a priori, less biased)
+ bins = len(v["df"])
+ index = int(round(bins*v["threshold"]))
+ index = min(index, len(v["df"])-1) #avoids out of range indexing
+ entry.append(v["df"].f1_score[index])
+ entry.append(v["df"].std_f1[index])
+ entry.append(v["df"].precision[index])
+ entry.append(v["df"].recall[index])
+
+ # statistics based on the best threshold (a posteriori, biased)
+ entry.append(v["df"].f1_score.max())
+ f1max_idx = v["df"].f1_score.idxmax()
+ entry.append(v["df"].precision[f1max_idx])
+ entry.append(v["df"].recall[f1max_idx])
+
+ table.append(entry)
+
+ return tabulate.tabulate(table, headers, tablefmt=fmt, floatfmt=".3f")
diff --git a/bob/ip/binseg/utils/transformfolder.py b/bob/ip/binseg/utils/transformfolder.py
deleted file mode 100644
index 9308d64705adeba36ae0edefc7f07b3f56b8069f..0000000000000000000000000000000000000000
--- a/bob/ip/binseg/utils/transformfolder.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from pathlib import Path,PurePosixPath
-from PIL import Image
-from torchvision.transforms.functional import to_pil_image
-
-def transformfolder(source_path, target_path, transforms):
- """Applies a set of transfroms on an image folder
-
- Parameters
- ----------
- source_path : str
- [description]
- target_path : str
- [description]
- transforms : [type]
- transform function
- """
- source_path = Path(source_path)
- target_path = Path(target_path)
- file_paths = sorted(list(source_path.glob('*?.*')))
- for f in file_paths:
- timg_path = PurePosixPath(target_path).joinpath(f.name)
- img = Image.open(f).convert(mode='1', dither=None)
- img, _ = transforms(img,img)
- img = to_pil_image(img)
- img.save(str(timg_path))
\ No newline at end of file
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 1a7c95acb6da05634579c283c8de6248c82a7677..4685c1b569c4be237642cdfcb5da41f1b8d77d58 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -25,32 +25,30 @@ requirements:
host:
- python {{ python }}
- setuptools {{ setuptools }}
- - torchvision {{ torchvision }} # [linux]
- - pytorch {{ pytorch }} # [linux]
- numpy {{ numpy }}
+ - h5py {{ h5py }}
+ - pytorch {{ pytorch }} # [linux]
+ - torchvision {{ torchvision }} # [linux]
- bob.extension
run:
- python
- setuptools
+ - {{ pin_compatible('numpy') }}
- {{ pin_compatible('pytorch') }} # [linux]
- {{ pin_compatible('torchvision') }} # [linux]
- - {{ pin_compatible('numpy') }}
- - pandas
- matplotlib
+ - pandas
+ - pillow
+ - psutil
+ - h5py
- tqdm
- tabulate
- - bob.core
test:
imports:
- {{ name }}
commands:
- # test commands ("script" entry-points) from your package here
- - bob binseg --help
- - bob binseg train --help
- - bob binseg test --help
- - bob binseg compare --help
- - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
+ - nosetests --with-coverage --cover-package={{ name }} --cover-erase --cover-html-dir={{ project_dir }}/sphinx/coverage --cover-html --cover-xml-file={{ project_dir }}/coverage.xml --cover-xml -sv {{ name }}
- sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
- sphinx-build -aEb doctest {{ project_dir }}/doc sphinx
- conda inspect linkages -p $PREFIX {{ name }} # [not win]
@@ -61,15 +59,8 @@ test:
- coverage
- sphinx
- sphinx_rtd_theme
- - bob.db.drive
- - bob.db.stare
- - bob.db.chasedb1
- - bob.db.hrf
- - bob.db.drionsdb
- - bob.db.rimoner3
- - bob.db.drishtigs1
- - bob.db.refuge
- - bob.db.iostar
+ - sphinxcontrib-programoutput
+ - graphviz
about:
summary: Binary Segmentation Benchmark Package for Bob
diff --git a/develop.cfg b/develop.cfg
index 9229bdf942a37e259b48ae2e724027d8b228f214..0bc61a9ea2ad1c927e7929eaf7f711c715a2e5a8 100644
--- a/develop.cfg
+++ b/develop.cfg
@@ -4,10 +4,6 @@
[buildout]
parts = scripts
eggs = bob.ip.binseg
- bob.db.drive
- bob.db.stare
- bob.db.chasedb1
- bob.db.hrf
bob.db.drionsdb
bob.db.rimoner3
bob.db.drishtigs1
@@ -16,11 +12,7 @@ eggs = bob.ip.binseg
extensions = bob.buildout
mr.developer
auto-checkout = *
-develop = src/bob.db.drive
- src/bob.db.stare
- src/bob.db.chasedb1
- src/bob.db.hrf
- src/bob.db.drionsdb
+develop = src/bob.db.drionsdb
src/bob.db.rimoner3
src/bob.db.drishtigs1
src/bob.db.refuge
@@ -33,10 +25,6 @@ verbose = true
newest = false
[sources]
-bob.db.drive = git git@gitlab.idiap.ch:bob/bob.db.drive
-bob.db.stare = git git@gitlab.idiap.ch:bob/bob.db.stare
-bob.db.chasedb1 = git git@gitlab.idiap.ch:bob/bob.db.chasedb1
-bob.db.hrf = git git@gitlab.idiap.ch:bob/bob.db.hrf
bob.db.drionsdb = git git@gitlab.idiap.ch:bob/bob.db.drionsdb
bob.db.rimoner3 = git git@gitlab.idiap.ch:bob/bob.db.rimoner3
bob.db.drishtigs1 = git git@gitlab.idiap.ch:bob/bob.db.drishtigs1
diff --git a/doc/_templates/config.rst b/doc/_templates/config.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a287392f705f1c10adec86c2e2e5a1b0b1c9cb24
--- /dev/null
+++ b/doc/_templates/config.rst
@@ -0,0 +1,2 @@
+{% include "autosummary/module.rst" %}
+.. literalinclude:: ../../../../{{ fullname.replace(".", "/") }}.py
diff --git a/doc/acknowledgements.rst b/doc/acknowledgements.rst
index 7dde8b5d62beb777bc0145af8ad5cbdbae53ebff..9783907ffca7da96f7dc391994894e079c11d1be 100644
--- a/doc/acknowledgements.rst
+++ b/doc/acknowledgements.rst
@@ -1,39 +1,17 @@
.. -*- coding: utf-8 -*-
+
.. _bob.ip.binseg.acknowledgements:
-================
-Acknowledgements
-================
+==================
+ Acknowledgements
+==================
This packages utilizes code from the following packages:
-* The model-checkpointer is based on the Checkpointer in maskrcnn_benchmark by::
-
- @misc{massa2018mrcnn,
- author = {Massa, Francisco and Girshick, Ross},
- title = {{maskrcnn-benchmark: Fast, modular reference implementation of Instance Segmentation and Object Detection algorithms in PyTorch}},
- year = {2018},
- howpublished = {\url{https://github.com/facebookresearch/maskrcnn-benchmark}},
- note = {Accessed: 2019.05.01}
- }
-
-* The AdaBound optimizer code by::
-
- @inproceedings{Luo2019AdaBound,
- author = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
- title = {Adaptive Gradient Methods with Dynamic Bound of Learning Rate},
- booktitle = {Proceedings of the 7th International Conference on Learning Representations},
- month = {May},
- year = {2019},
- address = {New Orleans, Louisiana}
- }
+* The model-checkpointer is based on the implementation in
+ `maskrcnn-benchmark`_ by [MASSA-2018]_
+* The AdaBound optimizer code was sourced from [LUO-2019]_
+* The MobileNetV2 backbone is based on [LIN-2018]_
-* The MobileNetV2 backbone is based on an implementation by::
- @misc{tonylins,
- author = {Ji Lin},
- title = {pytorch-mobilenet-v2},
- year = {2018}
- howpublished = {\url{https://github.com/tonylins/pytorch-mobilenet-v2}},
- note = {Accessed: 2019.05.01}
- }
+.. include:: links.rst
diff --git a/doc/api.rst b/doc/api.rst
index 1eade48273e9a1117484ac0d14a7df6b510b5e3d..8d145ad5534429edf24cc0be8286449ab86f93a9 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,50 +1,187 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.api:
-============
- Python API
-============
+=====================================
+ Application Program Interface (API)
+=====================================
-This section lists all the functionality available in this library allowing to
-run binary-segmentation benchmarks.
+.. To update these lists, run the following command on the root of the package:
+.. find bob -name '*.py' | sed -e 's#/#.#g;s#.py$##g;s#.__init__##g' | sort
+.. You may apply further filtering to update only one of the subsections below
-PyTorch bob.db Dataset
-======================
-.. automodule:: bob.ip.binseg.data.binsegdataset
+Data Manipulation
+-----------------
-PyTorch ImageFolder Dataset
-===========================
-.. automodule:: bob.ip.binseg.data.imagefolder
+.. autosummary::
+ :toctree: api/data
-.. automodule:: bob.ip.binseg.data.imagefolderinference
+ bob.ip.binseg.data.dataset
+ bob.ip.binseg.data.loader
+ bob.ip.binseg.data.sample
+ bob.ip.binseg.data.utils
+ bob.ip.binseg.data.transforms
-Transforms
-==========
-.. note::
- All transforms work with :py:class:`PIL.Image.Image` objects. We make heavy use of the
- `torchvision package`_
-.. automodule:: bob.ip.binseg.data.transforms
+Datasets
+--------
-Losses
+.. autosummary::
+ :toctree: api/dataset
+
+ bob.ip.binseg.data.drive
+ bob.ip.binseg.data.stare
+ bob.ip.binseg.data.chasedb1
+ bob.ip.binseg.data.hrf
+ bob.ip.binseg.data.iostar
+ bob.ip.binseg.data.refuge
+ bob.ip.binseg.data.drishtigs1
+ bob.ip.binseg.data.rimoner3
+ bob.ip.binseg.data.drionsdb
+
+
+Engines
+-------
+
+.. autosummary::
+ :toctree: api/engine
+
+ bob.ip.binseg.engine
+ bob.ip.binseg.engine.trainer
+ bob.ip.binseg.engine.ssltrainer
+ bob.ip.binseg.engine.predictor
+ bob.ip.binseg.engine.evaluator
+ bob.ip.binseg.engine.adabound
+
+
+Neural Network Models
+---------------------
+
+.. autosummary::
+ :toctree: api/modeling
+
+ bob.ip.binseg.modeling
+ bob.ip.binseg.modeling.backbones
+ bob.ip.binseg.modeling.backbones.mobilenetv2
+ bob.ip.binseg.modeling.backbones.resnet
+ bob.ip.binseg.modeling.backbones.vgg
+ bob.ip.binseg.modeling.driu
+ bob.ip.binseg.modeling.driubn
+ bob.ip.binseg.modeling.driuod
+ bob.ip.binseg.modeling.driupix
+ bob.ip.binseg.modeling.hed
+ bob.ip.binseg.modeling.losses
+ bob.ip.binseg.modeling.m2u
+ bob.ip.binseg.modeling.make_layers
+ bob.ip.binseg.modeling.resunet
+ bob.ip.binseg.modeling.unet
+
+
+Toolbox
+-------
+
+.. autosummary::
+ :toctree: api/utils
+
+ bob.ip.binseg.utils
+ bob.ip.binseg.utils.checkpointer
+ bob.ip.binseg.utils.metric
+ bob.ip.binseg.utils.model_serialization
+ bob.ip.binseg.utils.model_zoo
+ bob.ip.binseg.utils.plot
+ bob.ip.binseg.utils.table
+ bob.ip.binseg.utils.summary
+
+
+.. _bob.ip.binseg.configs:
+
+Preset Configurations
+---------------------
+
+Preset configurations for baseline systems
+
+This module contains preset configurations for baseline FCN architectures and
+datasets.
+
+
+Models
======
-.. automodule:: bob.ip.binseg.modeling.losses
-Training
-========
-.. automodule:: bob.ip.binseg.engine.trainer
+.. autosummary::
+ :toctree: api/configs/models
+ :template: config.rst
-Checkpointer
-============
-.. automodule:: bob.ip.binseg.utils.checkpointer
+ bob.ip.binseg.configs.models.driu
+ bob.ip.binseg.configs.models.driu_bn
+ bob.ip.binseg.configs.models.driu_bn_ssl
+ bob.ip.binseg.configs.models.driu_od
+ bob.ip.binseg.configs.models.driu_ssl
+ bob.ip.binseg.configs.models.hed
+ bob.ip.binseg.configs.models.m2unet
+ bob.ip.binseg.configs.models.m2unet_ssl
+ bob.ip.binseg.configs.models.resunet
+ bob.ip.binseg.configs.models.unet
-Inference and Evaluation
-========================
-.. automodule:: bob.ip.binseg.engine.inferencer
-Plotting
+.. _bob.ip.binseg.configs.datasets:
+
+Datasets
========
-.. automodule:: bob.ip.binseg.utils.plot
-.. include:: links.rst
+.. automodule:: bob.ip.binseg.configs.datasets
+
+.. autosummary::
+ :toctree: api/configs/datasets
+ :template: config.rst
+
+ bob.ip.binseg.configs.datasets.csv
+
+ bob.ip.binseg.configs.datasets.chasedb1.first_annotator
+ bob.ip.binseg.configs.datasets.chasedb1.second_annotator
+ bob.ip.binseg.configs.datasets.chasedb1.xtest
+ bob.ip.binseg.configs.datasets.chasedb1.mtest
+ bob.ip.binseg.configs.datasets.chasedb1.covd
+ bob.ip.binseg.configs.datasets.chasedb1.ssl
+
+ bob.ip.binseg.configs.datasets.drive.default
+ bob.ip.binseg.configs.datasets.drive.second_annotator
+ bob.ip.binseg.configs.datasets.drive.xtest
+ bob.ip.binseg.configs.datasets.drive.mtest
+ bob.ip.binseg.configs.datasets.drive.covd
+ bob.ip.binseg.configs.datasets.drive.ssl
+
+ bob.ip.binseg.configs.datasets.hrf.default
+ bob.ip.binseg.configs.datasets.hrf.xtest
+ bob.ip.binseg.configs.datasets.hrf.mtest
+ bob.ip.binseg.configs.datasets.hrf.default_fullres
+ bob.ip.binseg.configs.datasets.hrf.covd
+ bob.ip.binseg.configs.datasets.hrf.ssl
+
+ bob.ip.binseg.configs.datasets.iostar.vessel
+ bob.ip.binseg.configs.datasets.iostar.vessel_xtest
+ bob.ip.binseg.configs.datasets.iostar.vessel_mtest
+ bob.ip.binseg.configs.datasets.iostar.optic_disc
+ bob.ip.binseg.configs.datasets.iostar.covd
+ bob.ip.binseg.configs.datasets.iostar.ssl
+
+ bob.ip.binseg.configs.datasets.stare.ah
+ bob.ip.binseg.configs.datasets.stare.vk
+ bob.ip.binseg.configs.datasets.stare.xtest
+ bob.ip.binseg.configs.datasets.stare.mtest
+ bob.ip.binseg.configs.datasets.stare.covd
+ bob.ip.binseg.configs.datasets.stare.ssl
+
+ bob.ip.binseg.configs.datasets.refuge.cup
+ bob.ip.binseg.configs.datasets.refuge.disc
+
+ bob.ip.binseg.configs.datasets.rimoner3.cup_exp1
+ bob.ip.binseg.configs.datasets.rimoner3.cup_exp2
+ bob.ip.binseg.configs.datasets.rimoner3.disc_exp1
+ bob.ip.binseg.configs.datasets.rimoner3.disc_exp2
+
+ bob.ip.binseg.configs.datasets.drishtigs1.cup_all
+ bob.ip.binseg.configs.datasets.drishtigs1.cup_any
+ bob.ip.binseg.configs.datasets.drishtigs1.disc_all
+ bob.ip.binseg.configs.datasets.drishtigs1.disc_any
+
+ bob.ip.binseg.configs.datasets.drionsdb.expert1
+ bob.ip.binseg.configs.datasets.drionsdb.expert2
diff --git a/doc/benchmarkresults.rst b/doc/benchmarkresults.rst
deleted file mode 100644
index 2f391611db75358ffeb809058e5bf2242a474b93..0000000000000000000000000000000000000000
--- a/doc/benchmarkresults.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.benchmarkresults:
-
-
-==================
-Benchmark Results
-==================
-
-F1 Scores
-===========
-
-* Benchmark results for models: DRIU, HED, M2UNet and U-Net.
-* Models are trained and tested on the same dataset using the train-test split as indicated in :ref:`bob.ip.binseg.datasets`
-* Standard-deviations across all test images are indicated in brakets
-
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-| F1 (std) | :ref:`bob.ip.binseg.configs.datasets.chasedb1` | :ref:`bob.ip.binseg.configs.datasets.drive` | :ref:`bob.ip.binseg.configs.datasets.hrf` | :ref:`bob.ip.binseg.configs.datasets.iostar` | :ref:`bob.ip.binseg.configs.datasets.stare` |
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-| :ref:`bob.ip.binseg.configs.models.driu` | `0.810 (0.021) <driu_chasedb1.pth_>`_ | `0.820 (0.014) <driu_drive.pth_>`_ | `0.783 (0.055) <driu_hrf.pth_>`_ | `0.825 (0.020) <driu_iostar.pth_>`_ | `0.827 (0.037) <driu_stare.pth_>`_ |
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-| :ref:`bob.ip.binseg.configs.models.hed` | 0.810 (0.022) | 0.817 (0.013) | 0.783 (0.058) | 0.825 (0.020) | 0.823 (0.037) |
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-| :ref:`bob.ip.binseg.configs.models.m2unet` | `0.802 (0.019) <m2unet_chasedb1.pth_>`_ | `0.803 (0.014) <m2unet_drive.pth_>`_ | `0.780 (0.057) <m2unet_hrf.pth_>`_ | `0.817 (0.020) <m2unet_iostar.pth_>`_ | `0.815 (0.041) <m2unet_stare.pth_>`_ |
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-| :ref:`bob.ip.binseg.configs.models.unet` | 0.812 (0.020) | 0.822 (0.015) | 0.788 (0.051) | 0.818 (0.019) | 0.829 (0.042) |
-+--------------------------------------------+------------------------------------------------+---------------------------------------------+-------------------------------------------+----------------------------------------------+---------------------------------------------+
-
-
-
-.. include:: links.rst
diff --git a/doc/cli.rst b/doc/cli.rst
new file mode 100644
index 0000000000000000000000000000000000000000..588cc997c85333e3fd2cf948dd9bd650f26b8a8e
--- /dev/null
+++ b/doc/cli.rst
@@ -0,0 +1,182 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.cli:
+
+==============================
+ Command-Line Interface (CLI)
+==============================
+
+This package provides a single entry point for all of its applications using
+:ref:`Bob's unified CLI mechanism <bob.extension.cli>`. A list of available
+applications can be retrieved using:
+
+.. command-output:: bob binseg --help
+
+
+Setup
+-----
+
+A CLI application to list and check installed (raw) datasets.
+
+.. _bob.ip.binseg.cli.dataset:
+
+.. command-output:: bob binseg dataset --help
+
+
+List available datasets
+=======================
+
+Lists supported and configured raw datasets.
+
+.. _bob.ip.binseg.cli.dataset.list:
+
+.. command-output:: bob binseg dataset list --help
+
+
+Check available datasets
+========================
+
+Checks if we can load all files listed for a given dataset (all subsets in all
+protocols).
+
+.. _bob.ip.binseg.cli.dataset.check:
+
+.. command-output:: bob binseg dataset check --help
+
+
+Preset Configuration Resources
+------------------------------
+
+A CLI application allows one to list, inspect and copy available configuration
+resources exported by this package.
+
+.. _bob.ip.binseg.cli.config:
+
+.. command-output:: bob binseg config --help
+
+
+.. _bob.ip.binseg.cli.config.list:
+
+Listing Resources
+=================
+
+.. command-output:: bob binseg config list --help
+
+
+.. _bob.ip.binseg.cli.config.list.all:
+
+Available Resources
+===================
+
+Here is a list of all resources currently exported.
+
+.. command-output:: bob binseg config list -v
+
+
+.. _bob.ip.binseg.cli.config.describe:
+
+Describing a Resource
+=====================
+
+.. command-output:: bob binseg config describe --help
+
+
+.. _bob.ip.binseg.cli.config.copy:
+
+Copying a Resource
+==================
+
+You may use this command to locally copy a resource file so you can change it.
+
+.. command-output:: bob binseg config copy --help
+
+
+.. _bob.ip.binseg.cli.combined:
+
+Running and Analyzing Experiments
+---------------------------------
+
+These applications run a combined set of steps in one go. They work well with
+our preset :ref:`configuration resources <bob.ip.binseg.cli.config.list.all>`.
+
+
+.. _bob.ip.binseg.cli.experiment:
+
+Running a Full Experiment Cycle
+===============================
+
+This command can run training, prediction, evaluation and comparison from a
+single, multi-step application.
+
+.. command-output:: bob binseg experiment --help
+
+
+.. _bob.ip.binseg.cli.analyze:
+
+Running Complete Experiment Analysis
+====================================
+
+This command can run prediction, evaluation and comparison from a
+single, multi-step application.
+
+.. command-output:: bob binseg analyze --help
+
+
+.. _bob.ip.binseg.cli.single:
+
+Single-Step Applications
+------------------------
+
+These applications allow finer control over the experiment cycle. They also
+work well with our preset :ref:`configuration resources
+<bob.ip.binseg.cli.config.list.all>`, but allow finer control on the input
+datasets.
+
+
+.. _bob.ip.binseg.cli.train:
+
+Training FCNs
+=============
+
+Training creates of a new PyTorch_ model. This model can be used for
+evaluation tests or for inference.
+
+.. command-output:: bob binseg train --help
+
+
+.. _bob.ip.binseg.cli.predict:
+
+Prediction with FCNs
+====================
+
+Inference takes as input a PyTorch_ model and generates output probabilities as
+HDF5 files. The probability map has the same size as the input and indicates,
+from 0 to 1 (floating-point number), the probability of a vessel in that pixel,
+from less probable (0.0) to more probable (1.0).
+
+.. command-output:: bob binseg predict --help
+
+
+.. _bob.ip.binseg.cli.evaluate:
+
+FCN Performance Evaluation
+==========================
+
+Evaluation takes inference results and compares it to ground-truth, generating
+a series of analysis figures which are useful to understand model performance.
+
+.. command-output:: bob binseg evaluate --help
+
+
+.. _bob.ip.binseg.cli.compare:
+
+Performance Comparison
+======================
+
+Performance comparison takes the performance evaluation results and generate
+combined figures and tables that compare results of multiple systems.
+
+.. command-output:: bob binseg compare --help
+
+
+.. include:: links.rst
diff --git a/doc/conf.py b/doc/conf.py
index d64d7794ac7eaa26cc5ea7cb49728aa892946210..c822aa64808f039d7f956fef3044cada9fcd8d62 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -23,9 +23,15 @@ extensions = [
'sphinx.ext.napoleon',
'sphinx.ext.viewcode',
'sphinx.ext.mathjax',
+ 'sphinxcontrib.programoutput',
#'matplotlib.sphinxext.plot_directive'
]
+# This allows sphinxcontrib-programoutput to work in buildout mode
+candidate_binpath = os.path.join(os.path.dirname(os.path.realpath(os.curdir)), 'bin')
+if os.path.exists(candidate_binpath):
+ os.environ['PATH'] = candidate_binpath + os.pathsep + os.environ.get('PATH', '')
+
# Be picky about warnings
nitpicky = True
@@ -96,7 +102,12 @@ release = distribution.version
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
-exclude_patterns = ['links.rst']
+exclude_patterns = [
+ 'links.rst',
+ 'api/modules.rst',
+ 'api/bob.rst',
+ 'api/bob.ip.rst',
+ ]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
@@ -236,25 +247,12 @@ else:
intersphinx_mapping['torch'] = ('https://pytorch.org/docs/stable/', None)
intersphinx_mapping['PIL'] = ('http://pillow.readthedocs.io/en/stable', None)
intersphinx_mapping['pandas'] = ('https://pandas.pydata.org/pandas-docs/stable/',None)
-# We want to remove all private (i.e. _. or __.__) members
-# that are not in the list of accepted functions
-accepted_private_functions = ['__array__']
-
-
-def member_function_test(app, what, name, obj, skip, options):
- # test if we have a private function
- if len(name) > 1 and name[0] == '_':
- # test if this private function should be allowed
- if name not in accepted_private_functions:
- # omit privat functions that are not in the list of accepted private functions
- return skip
- else:
- # test if the method is documented
- if not hasattr(obj, '__doc__') or not obj.__doc__:
- return skip
- return False
-
-
-def setup(app):
- app.connect('autodoc-skip-member', member_function_test)
-
\ No newline at end of file
+
+# Figures out the major click version we use
+import pkg_resources
+click_version = pkg_resources.require('click')[0].version.split('.')[0]
+click_version += '.x'
+intersphinx_mapping['click'] = ('https://click.palletsprojects.com/en/%s/' % (click_version,),None)
+
+# Add our private index (for extras and fixes)
+intersphinx_mapping['extras'] = ('', 'extras.inv')
diff --git a/doc/configs.rst b/doc/configs.rst
deleted file mode 100644
index e25e66c6afc3e2fc629c915f25475d101f81033c..0000000000000000000000000000000000000000
--- a/doc/configs.rst
+++ /dev/null
@@ -1,208 +0,0 @@
-.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.configs:
-
-===============
-Configs
-===============
-
-Dataset Configs
-===============
-
-.. _bob.ip.binseg.configs.datasets.imagefolder:
-
-ImageFolder
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolder.py
-
-.. _bob.ip.binseg.configs.datasets.imagefoldertest:
-
-ImageFolderTest
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefoldertest.py
-
-.. _bob.ip.binseg.configs.datasets.imagefolderinference:
-
-ImageFolderInference
----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolderinference.py
-
-.. _bob.ip.binseg.configs.datasets.chasedb1:
-
-CHASEDB1
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1.py
-
-CHASEDB1TEST
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1test.py
-
-.. _bob.ip.binseg.configs.datasets.covd-drive:
-
-COVD-DRIVE
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py
-
-.. _bob.ip.binseg.configs.datasets.covd-drive_ssl:
-
-COVD-DRIVE_SSL
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py
-
-
-.. _bob.ip.binseg.configs.datasets.covd-stare:
-
-COVD-STARE
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py
-
-.. _bob.ip.binseg.configs.datasets.covd-stare_ssl:
-
-COVD-STARE_SSL
-----------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py
-
-
-.. _bob.ip.binseg.configs.datasets.covd-iostar:
-
-COVD-IOSTARVESSEL
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py
-
-.. _bob.ip.binseg.configs.datasets.covd-iostar_ssl:
-
-COVD-IOSTARVESSEL_SSL
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py
-
-
-.. _bob.ip.binseg.configs.datasets.covd-hrf:
-
-COVD-HRF
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py
-
-.. _bob.ip.binseg.configs.datasets.covd-hrf_ssl:
-
-COVD-HRF_SSL
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py
-
-
-.. _bob.ip.binseg.configs.datasets.covd-chasedb1:
-
-COVD-CHASEDB1
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
-
-.. _bob.ip.binseg.configs.datasets.covd-chasedb1_ssl:
-
-COVD-CHASEDB1_SSL
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
-
-
-.. _bob.ip.binseg.configs.datasets.drive:
-
-DRIVE
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drive.py
-
-
-DRIVETEST
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivetest.py
-
-
-.. _bob.ip.binseg.configs.datasets.hrf:
-
-HRF
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/hrf1168.py
-
-HRFTEST
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/hrftest.py
-
-
-
-.. _bob.ip.binseg.configs.datasets.iostar:
-
-IOSTARVESSEL
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvessel.py
-
-IOSTARVESSELTEST
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvesseltest.py
-
-
-
-.. _bob.ip.binseg.configs.datasets.stare:
-
-STARE
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/stare.py
-
-STARETEST
-----------------------
-.. literalinclude:: ../bob/ip/binseg/configs/datasets/staretest.py
-
-
-
-Model Configs
-==============
-
-.. _bob.ip.binseg.configs.models.driu:
-
-DRIU
------
-.. literalinclude:: ../bob/ip/binseg/configs/models/driu.py
-
-
-.. _bob.ip.binseg.configs.models.driubn:
-
-DRIUBN
-------
-.. literalinclude:: ../bob/ip/binseg/configs/models/driubn.py
-
-
-.. _bob.ip.binseg.configs.models.hed:
-
-HED
------
-.. literalinclude:: ../bob/ip/binseg/configs/models/hed.py
-
-
-.. _bob.ip.binseg.configs.models.m2unet:
-
-M2UNet
-------
-.. literalinclude:: ../bob/ip/binseg/configs/models/m2unet.py
-
-
-.. _bob.ip.binseg.configs.models.unet:
-
-UNet
------
-.. literalinclude:: ../bob/ip/binseg/configs/models/unet.py
-
-
-.. _bob.ip.binseg.configs.models.driussl:
-
-DRIUSSL
---------
-.. literalinclude:: ../bob/ip/binseg/configs/models/driussl.py
-
-.. _bob.ip.binseg.configs.models.driubnssl:
-
-DRIUBNSSL
----------
-.. literalinclude:: ../bob/ip/binseg/configs/models/driubnssl.py
-
-
-.. _bob.ip.binseg.configs.models.m2unetssl:
-
-M2UNetSSL
----------
-.. literalinclude:: ../bob/ip/binseg/configs/models/m2unetssl.py
-
diff --git a/doc/covdresults.rst b/doc/covdresults.rst
deleted file mode 100644
index bf2305f41fc0d8dfbb6fc379af11702d51b5e6da..0000000000000000000000000000000000000000
--- a/doc/covdresults.rst
+++ /dev/null
@@ -1,84 +0,0 @@
-.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.covdresults:
-
-
-==========================
-COVD- and COVD-SLL Results
-==========================
-
-In addition to the M2U-Net architecture, we also evaluated the larger DRIU network and a variation of it
-that contains batch normalization (DRIU BN) on COVD- and COVD-SSL. Perhaps surprisingly, for the
-majority of combinations, the performance of the DRIU variants are roughly equal or worse than the M2U-Net.
-We anticipate that one reason for this could be overparameterization of large VGG16 models
-that are pretrained on ImageNet.
-
-F1 Scores
-===========
-
-Comparison of F1-micro-scores (std) of DRIU and M2U-Net on COVD- and COVD-SSL.
-Standard deviation across test-images in brackets.
-
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| F1 score | :ref:`bob.ip.binseg.configs.models.driu`/:ref:`bob.ip.binseg.configs.models.driussl` | :ref:`bob.ip.binseg.configs.models.driubn`/:ref:`bob.ip.binseg.configs.models.driubnssl` | :ref:`bob.ip.binseg.configs.models.m2unet`/:ref:`bob.ip.binseg.configs.models.m2unetssl` |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-drive` | 0.788 (0.018) | 0.797 (0.019) | `0.789 (0.018) <m2unet_covd-drive.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-drive_ssl` | 0.785 (0.018) | 0.783 (0.019) | `0.791 (0.014) <m2unet_covd-drive_ssl.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-stare` | 0.778 (0.117) | 0.778 (0.122) | `0.812 (0.046) <m2unet_covd-stare.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-stare_ssl` | 0.788 (0.102) | 0.811 (0.074) | `0.820 (0.044) <m2unet_covd-stare_ssl.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-chasedb1` | 0.796 (0.027) | 0.791 (0.025) | `0.788 (0.024) <m2unet_covd-chasedb1.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-chasedb1_ssl` | 0.796 (0.024) | 0.798 (0.025) | `0.799 (0.026) <m2unet_covd-chasedb1_ssl.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-hrf` | 0.799 (0.044) | 0.800 (0.045) | `0.802 (0.045) <m2unet_covd-hrf.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-hrf_ssl` | 0.799 (0.044) | 0.784 (0.048) | `0.797 (0.044) <m2unet_covd-hrf_ssl.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-iostar` | 0.791 (0.021) | 0.777 (0.032) | `0.793 (0.015) <m2unet_covd-iostar.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-| :ref:`bob.ip.binseg.configs.datasets.covd-iostar_ssl` | 0.797 (0.017) | 0.811 (0.074) | `0.785 (0.018) <m2unet_covd-iostar_ssl.pth>`_ |
-+---------------------------------------------------------+--------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+
-
-M2U-Net Precision vs. Recall Curves
-===================================
-Precision vs. recall curves for each evaluated dataset.
-Note that here the F1-score is calculated on a macro level (see paper for more details).
-
-.. figure:: img/pr_CHASEDB1.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- CHASE_DB1: Precision vs Recall curve and F1 scores
-
-.. figure:: img/pr_DRIVE.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- DRIVE: Precision vs Recall curve and F1 scores
-
-.. figure:: img/pr_HRF.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- HRF: Precision vs Recall curve and F1 scores
-
-.. figure:: img/pr_IOSTARVESSEL.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- IOSTAR: Precision vs Recall curve and F1 scores
-
-.. figure:: img/pr_STARE.png
- :scale: 50 %
- :align: center
- :alt: model comparisons
-
- STARE: Precision vs Recall curve and F1 scores
-
diff --git a/doc/datasets.rst b/doc/datasets.rst
index 5b82d3b12f8909a414055a17a3360c73d8449e1e..a1fb29e04e27da12f0d508f15be1a498ff589c70 100644
--- a/doc/datasets.rst
+++ b/doc/datasets.rst
@@ -1,64 +1,129 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.datasets:
-
-==================
-Supported Datasets
-==================
-
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| # | Name | H x W | # imgs | Train | Test | Mask | Vessel | OD | Cup | Train-Test split reference |
-+=====+===============+=============+========+=======+======+======+========+=====+=====+============================+
-| 1 | Drive_ | 584 x 565 | 40 | 20 | 20 | x | x | | | `Staal et al. (2004)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 2 | STARE_ | 605 x 700 | 20 | 10 | 10 | | x | | | `Maninis et al. (2016)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 3 | CHASEDB1_ | 960 x 999 | 28 | 8 | 20 | | x | | | `Fraz et al. (2012)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 4 | HRF_ | 2336 x 3504 | 45 | 15 | 30 | x | x | | | `Orlando et al. (2016)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 5 | IOSTAR_ | 1024 x 1024 | 30 | 20 | 10 | x | x | x | | `Meyer et al. (2017)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 6 | DRIONS-DB_ | 400 x 600 | 110 | 60 | 50 | | | x | | `Maninis et al. (2016)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 7 | RIM-ONEr3_ | 1424 x 1072 | 159 | 99 | 60 | | | x | x | `Maninis et al. (2016)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 8 | Drishti-GS1_ | varying | 101 | 50 | 51 | | | x | x | `Sivaswamy et al. (2014)`_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 9 | REFUGE_ train | 2056 x 2124 | 400 | 400 | | | | x | x | REFUGE_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-| 9 | REFUGE_ val | 1634 x 1634 | 400 | | 400 | | | x | x | REFUGE_ |
-+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
-
-
-Add-on: Folder-based Dataset
-============================
-
-For quick experimentation we also provide a PyTorch class that works with the following
-dataset folder structure for images and ground-truth (gt):
-
-.. code-block:: bash
-
- root
- |- images
- |- gt
-the file names should have the same stem. Currently all image formats that can be read via PIL are supported. Additionally we support hdf5 binary files.
-
-For training a new dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefolder` and amend accordingly,
-e.g. the full path of the dataset and if necessary any preprocessing steps such as resizing, cropping, padding etc..
-
-Training can then be started with
-
-.. code-block:: bash
-
- bob binseg train M2UNet /path/to/myimagefolderconfig.py -b 4 -d cuda -o /my/output/path -vv
-
-Similary for testing, a test dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefoldertest` and amend accordingly.
-
-Testing can then be started with
+.. _bob.ip.binseg.datasets:
-.. code-block:: bash
+====================
+ Supported Datasets
+====================
+
+Here is a list of currently support datasets in this package, alongside notable
+properties. Each dataset name is linked to the current location where raw data
+can be downloaded. We include the reference of the data split protocols used
+to generate iterators for training and testing.
+
+
+.. list-table::
+
+ * - Dataset
+ - Reference
+ - H x W
+ - Samples
+ - Mask
+ - Vessel
+ - OD
+ - Cup
+ - Split Reference
+ - Train
+ - Test
+ * - DRIVE_
+ - [DRIVE-2004]_
+ - 584 x 565
+ - 40
+ - x
+ - x
+ -
+ -
+ - [DRIVE-2004]_
+ - 20
+ - 20
+ * - STARE_
+ - [STARE-2000]_
+ - 605 x 700
+ - 20
+ -
+ - x
+ -
+ -
+ - [MANINIS-2016]_
+ - 10
+ - 10
+ * - CHASE-DB1_
+ - [CHASEDB1-2012]_
+ - 960 x 999
+ - 28
+ -
+ - x
+ -
+ -
+ - [CHASEDB1-2012]_
+ - 8
+ - 20
+ * - HRF_
+ - [HRF-2013]_
+ - 2336 x 3504
+ - 45
+ - x
+ - x
+ -
+ -
+ - [ORLANDO-2017]_
+ - 15
+ - 30
+ * - IOSTAR_
+ - [IOSTAR-2016]_
+ - 1024 x 1024
+ - 30
+ - x
+ - x
+ - x
+ -
+ - [MEYER-2017]_
+ - 20
+ - 10
+ * - DRIONS-DB_
+ - [DRIONSDB-2008]_
+ - 400 x 600
+ - 110
+ -
+ -
+ - x
+ -
+ - [MANINIS-2016]_
+ - 60
+ - 50
+ * - `RIM-ONE r3`_
+ - [RIMONER3-2015]_
+ - 1424 x 1072
+ - 159
+ -
+ -
+ - x
+ - x
+ - [MANINIS-2016]_
+ - 99
+ - 60
+ * - Drishti-GS1_
+ - [DRISHTIGS1-2014]_
+ - varying
+ - 101
+ -
+ -
+ - x
+ - x
+ - [DRISHTIGS1-2014]_
+ - 50
+ - 51
+ * - REFUGE_
+ - [REFUGE-2018]_
+ - 2056 x 2124 (1634 x 1634)
+ - 1200
+ -
+ -
+ - x
+ - x
+ - [REFUGE-2018]_
+ - 400 (+400)
+ - 400
- bob binseg test M2UNet /path/to/myimagefoldertestconfig.py -b 2 -d cuda -o /my/output/path -vv
.. include:: links.rst
diff --git a/doc/evaluation.rst b/doc/evaluation.rst
index 2fb923a00978a3a6aa172722697472f77c94a118..0368ad6ddd23ab5c231a0e3c29fe8f88f9c0e51b 100644
--- a/doc/evaluation.rst
+++ b/doc/evaluation.rst
@@ -1,111 +1,97 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.evaluation:
-==========
-Evaluation
-==========
+.. _bob.ip.binseg.eval:
-To evaluate trained models use use ``bob binseg test`` followed by
-the model config, the dataset config and the path to the pretrained
-model via the argument ``-w``.
+==========================
+ Inference and Evaluation
+==========================
-Alternatively point to the output folder used during training via
-the ``-o`` argument. The Checkpointer will load the model as indicated
-in the file: ``last_checkpoint``.
+This guides explains how to run inference or a complete evaluation using
+command-line tools. Inference produces probability maps for input images,
+while evaluation will analyze such output against existing annotations and
+produce performance figures.
-Use ``bob binseg test --help`` for more information.
-E.g. run inference on model M2U-Net on the DRIVE test set:
+Inference
+---------
-.. code-block:: bash
+You may use one of your trained models (or :ref:`one of ours
+<bob.ip.binseg.models>` to run inference on existing datasets or your own
+dataset. In inference (or prediction) mode, we input data, the trained model,
+and output HDF5 files containing the prediction outputs for every input image.
+Each HDF5 file contains a single object with a 2-dimensional matrix of floating
+point numbers indicating the vessel probability (``[0.0,1.0]``) for each pixel
+in the input image.
- # Point directly to saved model via -w argument:
- bob binseg test M2UNet DRIVETEST -o /outputfolder/for/results -w /direct/path/to/weight/model_final.pth
- # Use training output path (requries last_checkpoint file to be present)
- # The evaluation results will be stored in the same folder
- bob binseg test M2UNet DRIVETEST -o /DRIVE/M2UNet/output
+Inference on an existing dataset
+================================
+
+To run inference, use the sub-command :ref:`predict
+<bob.ip.binseg.cli.predict>` to run prediction on an existing dataset:
+
+.. code-block:: sh
+
+ $ bob binseg predict -vv <model> -w <path/to/model.pth> <dataset>
-Outputs
-========
-The inference run generates the following output files:
+
+Replace ``<model>`` and ``<dataset>`` by the appropriate :ref:`configuration
+files <bob.ip.binseg.configs>`. Replace ``<path/to/model.pth>`` to a path
+leading to the pre-trained model, or URL pointing to a pre-trained model (e.g.
+:ref:`one of ours <bob.ip.binseg.models>`).
+
+
+Inference on a custom dataset
+=============================
+
+If you would like to test your own data against one of the pre-trained models,
+you need to instantiate :py:mod:`A CSV-based configuration
+<bob.ip.binseg.configs.datasets.csv>`
+
+Read the appropriate module documentation for details.
.. code-block:: bash
- .
- ├── images # the predicted probabilities as grayscale images in .png format
- ├── hdf5 # the predicted probabilties in hdf5 format
- ├── last_checkpoint # text file that keeps track of the last checkpoint
- ├── M2UNet_trainlog.csv # training log
- ├── M2UNet_trainlog.pdf # training log plot
- ├── model_*.pth # model checkpoints
- └── results
- ├── image*.jpg.csv # evaluation metrics for each image
- ├── Metrics.csv # average evaluation metrics
- ├── ModelSummary.txt # model summary and parameter count
- ├── precision_recall.pdf # precision vs recall plot
- └── Times.txt # inference times
-
-Inference Only Mode
-====================
-
-If you wish to run inference only on a folder containing images, use the ``predict`` function in combination with a :ref:`bob.ip.binseg.configs.datasets.imagefolderinference` config. E.g.:
+ $ bob binseg config copy csv-dataset-example mydataset.py
+ # edit mydataset.py to your liking
+ $ bob binseg predict -vv <model> -w <path/to/model.pth> ./mydataset.py
+
+
+Inference typically consumes less resources than training, but you may speed
+things up using ``--device='cuda:0'`` in case you have a GPU.
+
+
+Evaluation
+----------
+
+In evaluation, we input an **annotated** dataset and predictions to generate
+performance summaries that help analysis of a trained model. Evaluation is
+done using the :ref:`evaluate command `<bob.ip.binseg.cli.evaluate>` followed
+by the model and the annotated dataset configuration, and the path to the
+pretrained weights via the ``--weight`` argument.
+
+Use ``bob binseg evaluate --help`` for more information.
+
+E.g. run inference on predictions from the DRIVE test set, do the following:
.. code-block:: bash
- bob binseg predict M2UNet /path/to/myinferencedatasetconfig.py -b 1 -d cpu -o /my/output/path -w /path/to/pretrained/weight/model_final.pth -vv
+ # Point directly to saved model via -w argument:
+ bob binseg evaluate -vv drive-test -p /predictions/folder -o /eval/results/folder
+
+If available, you may use the option ``--second-annotator`` to
-Pretrained Models
+
+Comparing Systems
=================
-Due to storage limitations we only provide weights of a subset
-of all evaluated models:
-
-
-
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| | DRIU | M2UNet |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| DRIVE | `DRIU_DRIVE.pth`_ | `M2UNet_DRIVE.pth <m2unet_drive.pth_>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-DRIVE | | `M2UNet_COVD-DRIVE.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-DRIVE.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-DRIVE SSL | | `M2UNet_COVD-DRIVE_SSL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-DRIVE_SSL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| STARE | DRIU_STARE.pth_ | `M2UNet_STARE.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_STARE.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-STARE | | `M2UNet_COVD-STARE.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-STARE.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-STARE SSL | | `M2UNet_COVD-STARE_SSL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-STARE_SSL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| CHASE_DB1 | DRIU_CHASEDB1.pth_ | `M2UNet_CHASEDB1.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_CHASEDB1.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-CHASE_DB1 | | `M2UNet_COVD-CHASEDB1.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-CHASE_DB1 SSL | | `M2UNet_COVD-CHASEDB1_SSL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1_SSL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| IOSTARVESSEL | DRIU_IOSTAR.pth_ | `M2UNet_IOSTARVESSEL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_IOSTARVESSEL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-IOSTAR | | `M2UNet_COVD-IOSTAR.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-IOSTAR.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-IOSTAR SSL | | `M2UNet_COVD-IOSTAR_SSL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-IOSTAR_SSL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| HRF | DRIU_HRF.pth_ | `M2UNet_HRF1168.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_HRF1168.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-HRF | | `M2UNet_COVD-HRF.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-| COVD-HRF SSL | | `M2UNet_COVD-HRF_SSL.pth <https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF_SSL.pth>`_ |
-+--------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+
-
-
-
-To run evaluation of pretrained models pass url as ``-w`` argument. E.g.:
+To compare multiple systems together and generate combined plots and tables,
+use the :ref:`compare command <bob.ip.binseg.cli.compare>`. Use ``--help`` for
+a quick guide.
.. code-block:: bash
- bob binseg test DRIU DRIVETEST -o Evaluation_DRIU_DRIVE -w https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_DRIVE.pth
- bob binseg test M2UNet DRIVETEST -o Evaluation_M2UNet_DRIVE -w https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_DRIVE.pth
-
+ $ bob binseg compare -vv A A/metrics.csv B B/metrics.csv
.. include:: links.rst
diff --git a/doc/experiment.rst b/doc/experiment.rst
new file mode 100644
index 0000000000000000000000000000000000000000..9050af004419e2ca7fb3a2d41a72c245bcc448eb
--- /dev/null
+++ b/doc/experiment.rst
@@ -0,0 +1,192 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.experiment:
+
+==============================
+ Running complete experiments
+==============================
+
+We provide an :ref:`aggregator command called "experiment"
+<bob.ip.binseg.cli.experiment>` that runs training, followed by prediction,
+evaluation and comparison. After running, you will be able to find results
+from model fitting, prediction, evaluation and comparison under a single output
+directory.
+
+For example, to train a Mobile V2 U-Net architecture on the STARE dataset,
+evaluate both train and test set performances, output prediction maps and
+overlay analysis, together with a performance curve, run the following:
+
+.. code-block:: sh
+
+ $ bob binseg experiment -vv m2unet stare --batch-size=16 --overlayed
+ # check results in the "results" folder
+
+You may run the system on a GPU by using the ``--device=cuda:0`` option.
+
+
+Using your own dataset
+======================
+
+To use your own dataset, we recommend you read our instructions at
+:py:mod:`bob.ip.binseg.configs.datasets.csv`, and setup one or more CSV file
+describing input data and ground-truth (segmentation maps), and potential test
+data. Then, prepare a configuration file by copying our configuration example
+and edit it to apply the required transforms to your input data. Once you are
+happy with the result, use it in place of one of our datasets:
+
+.. code-block:: sh
+
+ $ bob binseg config copy csv-dataset-example mydataset.py
+ # edit mydataset following instructions
+ $ bob binseg train ... mydataset.py ...
+
+
+Baseline Benchmarks
+===================
+
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for supervised training of baselines. Use it like this:
+
+.. code-block:: sh
+
+ # change <model> and <dataset> by one of items bellow
+ $ bob binseg experiment -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+ # check results in the "results" folder
+
+.. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
+ - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
+ - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
+ - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
+ - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`
+ * - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
+ - 4
+ - 2
+ - 2
+ - 2
+ - 1
+ * - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
+ - 8
+ - 4
+ - 4
+ - 4
+ - 1
+ * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
+ - 8
+ - 5
+ - 4
+ - 4
+ - 1
+ * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - 16
+ - 6
+ - 6
+ - 6
+ - 1
+
+
+.. tip::
+
+ Instead of the default configurations, you can pass the full path of your
+ customized dataset and model files. You may :ref:`copy any of the existing
+ configuration resources <bob.ip.binseg.cli.config.copy>` and change them
+ locally. Once you're happy, you may use the newly created files directly on
+ your command line. For example, suppose you wanted to slightly change the
+ DRIVE pre-processing pipeline. You could do the following:
+
+ .. code-block:: bash
+
+ $ bob binseg config copy drive my_drive_remix.py
+ # edit my_drive_remix.py to your needs
+ $ bob binseg train -vv <model> ./my_drive_remix.py
+
+
+.. _bob.ip.binseg.gridtk-tip:
+
+.. tip::
+
+ If you are at Idiap, you may install the package ``gridtk`` (``conda install
+ gridtk``) on your environment, and submit the job like this:
+
+ .. code-block:: sh
+
+ $ jman submit --queue=gpu --memory=24G --name=myjob -- bob binseg train --device='cuda:0' ... #paste the rest of the command-line
+
+.. _bob.ip.binseg.baseline-script:
+
+The :download:`following shell script <scripts/baselines.sh>` can run the
+various baselines described above and place results in a single directory:
+
+.. literalinclude:: scripts/baselines.sh
+ :language: bash
+
+You will find results obtained running these baselines :ref:`further in this
+guide <bob.ip.binseg.results.baselines>`.
+
+
+Combined Vessel Dataset (COVD)
+==============================
+
+The following table describes recommended batch sizes for 24Gb of RAM GPU card,
+for supervised training of COVD- systems. Use it like this:
+
+.. code-block:: sh
+
+ # change <model> and <dataset> by one of items bellow
+ $ bob binseg train -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive-covd <bob.ip.binseg.configs.datasets.drive.covd>`
+ - :py:mod:`stare-covd <bob.ip.binseg.configs.datasets.stare.covd>`
+ - :py:mod:`chasedb1-covd <bob.ip.binseg.configs.datasets.chasedb1.covd>`
+ - :py:mod:`iostar-vessel-covd <bob.ip.binseg.configs.datasets.iostar.covd>`
+ - :py:mod:`hrf-covd <bob.ip.binseg.configs.datasets.hrf.covd>`
+ * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driu_bn>`
+ - 4
+ - 4
+ - 2
+ - 2
+ - 2
+ * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - 8
+ - 4
+ - 4
+ - 4
+ - 4
+
+
+Combined Vessel Dataset (COVD) and Semi-Supervised Learning (SSL)
+=================================================================
+
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for semi-supervised learning of COVD- systems. Use it like this:
+
+.. code-block:: sh
+
+ # change <model> and <dataset> by one of items bellow
+ $ bob binseg train -vv --ssl <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+ * - **Models / Datasets**
+ - :py:mod:`drive-ssl <bob.ip.binseg.configs.datasets.drive.ssl>`
+ - :py:mod:`stare-ssl <bob.ip.binseg.configs.datasets.stare.ssl>`
+ - :py:mod:`chasedb1-ssl <bob.ip.binseg.configs.datasets.chasedb1.ssl>`
+ - :py:mod:`iostar-vessel-ssl <bob.ip.binseg.configs.datasets.iostar.ssl>`
+ - :py:mod:`hrf-ssl <bob.ip.binseg.configs.datasets.hrf.ssl>`
+ * - :py:mod:`driu-ssl <bob.ip.binseg.configs.models.driu_ssl>` / :py:mod:`driu-bn-ssl <bob.ip.binseg.configs.models.driu_bn_ssl>`
+ - 4
+ - 4
+ - 2
+ - 1
+ - 1
+ * - :py:mod:`m2unet-ssl <bob.ip.binseg.configs.models.m2unet_ssl>`
+ - 4
+ - 4
+ - 2
+ - 2
+ - 2
diff --git a/doc/extra-intersphinx.txt b/doc/extra-intersphinx.txt
index 37f700a78ecad7bea22172702e640797d7136a17..ac988bdf8417d99a3c45236479862b18684c79f5 100644
--- a/doc/extra-intersphinx.txt
+++ b/doc/extra-intersphinx.txt
@@ -1,2 +1,2 @@
torch
-torchvision
\ No newline at end of file
+torchvision
diff --git a/doc/extras.inv b/doc/extras.inv
new file mode 100644
index 0000000000000000000000000000000000000000..55baaba61aa2392fa690178e23d9dbea964b8fc4
Binary files /dev/null and b/doc/extras.inv differ
diff --git a/doc/extras.txt b/doc/extras.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4bd227b7d23d3ad0c631ffc5a285e08378c44468
--- /dev/null
+++ b/doc/extras.txt
@@ -0,0 +1,20 @@
+# Sphinx inventory version 2
+# Project: extras
+# Version: stable
+# The remainder of this file is compressed using zlib.
+torch.optim.optimizer.Optimizer py:class 1 https://pytorch.org/docs/stable/optim.html#torch.optim.Optimizer -
+torch.nn.Module py:class 1 https://pytorch.org/docs/stable/nn.html?highlight=module#torch.nn.Module -
+torch.nn.modules.module.Module py:class 1 https://pytorch.org/docs/stable/nn.html?highlight=module#torch.nn.Module -
+torch.utils.data.dataset.Dataset py:class 1 https://pytorch.org/docs/stable/data.html?highlight=dataset#torch.utils.data.Dataset -
+unittest.case.TestCase py:class 1 https://docs.python.org/3/library/unittest.html?highlight=testcase#unittest.TestCase -
+click.core.Option py:class 1 https://click.palletsprojects.com/en/7.x/api/#click.Option -
+torchvision.transforms.transforms.ColorJitter py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.ColorJitter -
+torchvision.transforms.transforms.RandomRotation py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.RandomRotation -
+torchvision.transforms.transforms.RandomVerticalFlip py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.RandomVerticalFlip -
+torchvision.transforms.transforms.RandomHorizontalFlip py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.RandomHorizontalFlip -
+torchvision.transforms.transforms.Compose py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.Compose -
+torchvision.transforms.transforms.ToTensor py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.ToTensor -
+torchvision.transforms.transforms.Resize py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.Resize -
+torchvision.transforms.transforms.Pad py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.Pad -
+torchvision.transforms.transforms.CenterCrop py:class 1 https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.CenterCrop -
+torchvision.transforms py:module 1 https://pytorch.org/docs/stable/torchvision/transforms.html -
diff --git a/doc/framework.dot b/doc/framework.dot
new file mode 100644
index 0000000000000000000000000000000000000000..50bfce5a2ff0030ee75c0138efd884c4e8f3228b
--- /dev/null
+++ b/doc/framework.dot
@@ -0,0 +1,60 @@
+digraph framework {
+
+ graph [
+ rankdir=LR,
+ ];
+ edge [
+ fontname=Helvetica,
+ fontsize=12,
+ fontcolor=blue,
+ minlen=2,
+ labeldistance=2.5,
+ ];
+
+ node [
+ fontname=Helvetica,
+ fontsize=12,
+ fontcolor=black,
+ shape=record,
+ style="filled,rounded",
+ fillcolor=grey92,
+ ];
+
+ dataset [
+ label="<train>\nTraining\n\n|<test>\nTest\n\n",
+ fillcolor=yellow,
+ style="filled",
+ ];
+
+ {rank = min; dataset;}
+
+ subgraph cluster_experiment {
+ label=<<b>experiment</b>>;
+ shape=record;
+ style="filled,rounded";
+ fillcolor=white;
+ train;
+
+ subgraph cluster_analyze {
+ label=<<b>analyze</b>>;
+ predict;
+ evaluate;
+ compare;
+ }
+ }
+
+ figure, table [
+ fillcolor=lightblue,
+ style="filled",
+ ];
+ {rank = max; figure; table; }
+
+ dataset:train -> train [headlabel="sample + label",labelangle=30];
+ dataset:test -> predict [headlabel="sample",labelangle=30];
+ train -> predict [headlabel="model"];
+ dataset:test -> evaluate [headlabel="label"];
+ predict -> evaluate [headlabel="probabilities ",labelangle=-30];
+ evaluate -> compare [headlabel="metrics"];
+ compare -> figure;
+ compare -> table;
+}
diff --git a/doc/index.rst b/doc/index.rst
index c9d3853ae3eaaf8c1b4d837cb268fdef12dba6c7..b08fd5b7d3b0308bf39828076d4ee814971e1104 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -15,7 +15,7 @@ Please use the BibTeX reference below to cite this work:
.. code:: bibtex
@misc{laibacher_anjos_2019,
- title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Funduscopy},
+ title = {On the Evaluation and Real-World Usage Scenarios of Deep Vessel Segmentation for Retinography},
author = {Tim Laibacher and Andr\'e Anjos},
year = {2019},
eprint = {1909.03856},
@@ -26,30 +26,37 @@ Please use the BibTeX reference below to cite this work:
Additional Material
-===================
+-------------------
The additional material referred to in the paper can be found under
-:ref:`bob.ip.binseg.covdresults` and :download:`here </additionalresults.pdf>`
+:ref:`bob.ip.binseg.results` and :download:`here </additionalresults.pdf>`
-Users Guide
-===========
+.. todolist::
+
+
+User Guide
+----------
.. toctree::
:maxdepth: 2
setup
+ usage
+ results/index
+ acknowledgements
+ references
datasets
- training
- evaluation
- benchmarkresults
- covdresults
- configs
- plotting
- visualization
+ cli
api
- acknowledgements
-.. todolist::
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
.. include:: links.rst
diff --git a/doc/links.rst b/doc/links.rst
index 12e8c5d3e7d6f2f144f888b8d2c3a2dde515e406..ce2c72d68eb7f9a94b4704b39e30cbb3da111552 100644
--- a/doc/links.rst
+++ b/doc/links.rst
@@ -4,85 +4,77 @@
.. _idiap: http://www.idiap.ch
.. _bob: http://www.idiap.ch/software/bob
-.. _installation: https://www.idiap.ch/software/bob/docs/bob/docs/stable/bob/bob/doc/install.html
+.. _installation: https://www.idiap.ch/software/bob/install
.. _mailing list: https://www.idiap.ch/software/bob/discuss
-.. _torchvision package: https://github.com/pytorch/vision
-
-.. DRIVE
-
-.. _drive: https://doi.org/10.1109/TMI.2004.825627
-.. _staal et al. (2004): https://doi.org/10.1109/TMI.2004.825627
-
-.. STARE
-
-.. _stare: https://doi.org/10.1109/42.845178
-.. _maninis et al. (2016): https://doi.org/10.1007/978-3-319-46723-8_17
-
-.. HRF
-
-.. _hrf: http://dx.doi.org/10.1155/2013/154860
-.. _orlando et al. (2016): https://doi.org/10.1109/TBME.2016.2535311
-
-.. IOSTAR
-
-.. _iostar: https://doi.org/10.1109/TMI.2016.2587062
-.. _meyer et al. (2017): https://doi.org/10.1007/978-3-319-59876-5_56
-
-.. CHASEDB1
-
-.. _chasedb1: https://doi.org/10.1109/TBME.2012.2205687
-.. _fraz et al. (2012): https://doi.org/10.1109/TBME.2012.2205687
-
-.. DRIONSDB
-
-.. _drions-db: http://dx.doi.org/10.1016/j.artmed.2008.04.005
-.. _maninis et al. (2016): https://doi.org/10.1007/978-3-319-46723-8_17
-
-.. RIM-ONE r3
-
-.. _rim-oner3: https://dspace5.zcu.cz/bitstream/11025/29670/1/Fumero.pdf
-.. _maninis et al. (2016): https://doi.org/10.1007/978-3-319-46723-8_17
-
-.. Drishti-GS1
-
-.. _drishti-gs1: https://doi.org/10.1109/ISBI.2014.6867807
-.. _sivaswamy et al. (2014): https://doi.org/10.1109/ISBI.2014.6867807
-
-.. REFUGE
-
-.. _refuge: http://ai.baidu.com/broad/download?dataset=gon
-
-.. OtherPapers
-
-.. _Iglovikov et al. (2018): http://openaccess.thecvf.com/content_cvpr_2018_workshops/w4/html/Iglovikov_TernausNetV2_Fully_Convolutional_CVPR_2018_paper.html
-.. _He et al. (2015): https://doi.org/10.1109/ICCV.2015.164
+.. _pytorch: https://pytorch.org
+.. _tabulate: https://pypi.org/project/tabulate/
+.. _our paper: https://arxiv.org/abs/1909.03856
+
+.. Raw data websites
+.. _drive: https://www.isi.uu.nl/Research/Databases/DRIVE/
+.. _stare: http://cecas.clemson.edu/~ahoover/stare/
+.. _hrf: https://www5.cs.fau.de/research/data/fundus-images/
+.. _iostar: http://www.retinacheck.org/datasets
+.. _chase-db1: https://blogs.kingston.ac.uk/retinal/chasedb1/
+.. _drions-db: http://www.ia.uned.es/~ejcarmona/DRIONS-DB.html
+.. _rim-one r3: http://medimrg.webs.ull.es/research/downloads/
+.. _drishti-gs1: http://cvit.iiit.ac.in/projects/mip/drishti-gs/mip-dataset2/Home.php
+.. _refuge: https://refuge.grand-challenge.org/Details/
.. Software Tools
-
.. _maskrcnn-benchmark: https://github.com/facebookresearch/maskrcnn-benchmark
.. Pretrained models
-.. _driu_chasedb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_CHASEDB1.pth
-.. _driu_drive.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_DRIVE.pth
-.. _driu_hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_HRF1168.pth
-.. _driu_stare.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_STARE.pth
-.. _driu_iostar.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_IOSTARVESSEL.pth
+.. _baselines_driu_drive: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/driu/drive/model.pth
+.. _baselines_hed_drive: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/hed/drive/model.pth
+.. _baselines_m2unet_drive: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/m2unet/drive/model.pth
+.. _baselines_unet_drive: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/unet/drive/model.pth
+.. _baselines_driu_stare: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/driu/stare/model.pth
+.. _baselines_hed_stare: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/hed/stare/model.pth
+.. _baselines_m2unet_stare: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/m2unet/stare/model.pth
+.. _baselines_unet_stare: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/unet/stare/model.pth
+.. _baselines_driu_chase: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/driu/chasedb1/model.pth
+.. _baselines_hed_chase: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/hed/chasedb1/model.pth
+.. _baselines_m2unet_chase: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/m2unet/chasedb1/model.pth
+.. _baselines_unet_chase: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/unet/chasedb1/model.pth
+.. _baselines_driu_hrf: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/driu/hrf/model.pth
+.. _baselines_hed_hrf: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/hed/hrf/model.pth
+.. _baselines_m2unet_hrf: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/m2unet/hrf/model.pth
+.. _baselines_unet_hrf: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/unet/hrf/model.pth
+.. _baselines_driu_iostar: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/driu/iostar-vessel/model.pth
+.. _baselines_hed_iostar: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/hed/iostar-vessel/model.pth
+.. _baselines_m2unet_iostar: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/m2unet/iostar-vessel/model.pth
+.. _baselines_unet_iostar: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/baselines/unet/iostar-vessel/model.pth
-.. _m2unet_chasedb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_CHASEDB1.pth
+
+.. DRIVE
+.. _driu_drive.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_DRIVE.pth
.. _m2unet_drive.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_DRIVE.pth
-.. _m2unet_hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_HRF1168.pth
-.. _m2unet_stare.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_STARE.pth
-.. _m2unet_iostar.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_IOSTARVESSEL.pth
.. _m2unet_covd-drive.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-DRIVE.pth
.. _m2unet_covd-drive_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-DRIVE_SSL.pth
+
+.. STARE
+.. _driu_stare.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_STARE.pth
+.. _m2unet_stare.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_STARE.pth
.. _m2unet_covd-stare.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-STARE.pth
.. _m2unet_covd-stare_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-STARE_SSL.pth
-.. _m2unet_covd-chaesdb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1.pth
-.. _m2unet_covd-chaesdb1_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1_SSL.pth
-.. _m2unet_covd-hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF.pth
-.. _m2unet_covd-hrf_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF_SSL.pth
+
+.. CHASE-DB1
+.. _driu_chasedb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_CHASEDB1.pth
+.. _m2unet_chasedb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_CHASEDB1.pth
+.. _m2unet_covd-chasedb1.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1.pth
+.. _m2unet_covd-chasedb1_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-CHASEDB1_SSL.pth
+
+.. IOSTAR
+.. _driu_iostar.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_IOSTARVESSEL.pth
+.. _m2unet_iostar.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_IOSTARVESSEL.pth
.. _m2unet_covd-iostar.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-IOSTAR.pth
.. _m2unet_covd-iostar_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-IOSTAR_SSL.pth
+.. HRF
+.. _driu_hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/DRIU_HRF1168.pth
+.. _m2unet_hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_HRF1168.pth
+.. _m2unet_covd-hrf.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF.pth
+.. _m2unet_covd-hrf_ssl.pth: https://www.idiap.ch/software/bob/data/bob/bob.ip.binseg/master/M2UNet_COVD-HRF_SSL.pth
diff --git a/doc/models.rst b/doc/models.rst
new file mode 100644
index 0000000000000000000000000000000000000000..168b2414ad72cc730220315ca08ef6cb0a38411a
--- /dev/null
+++ b/doc/models.rst
@@ -0,0 +1,65 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.models:
+
+===================
+ Pretrained Models
+===================
+
+We offer the following pre-trained models allowing inference and score
+reproduction of our results. Due to storage limitations we only provide
+weights of a subset of all evaluated models.
+
+
+.. list-table::
+
+ * - **Datasets / Models**
+ - :py:mod:`driu <bob.ip.binseg.configs.models.driu>`
+ - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
+ - driu_drive.pth_
+ - m2unet_drive.pth_
+ * - :py:mod:`drive-drive <bob.ip.binseg.configs.datasets.drive.covd>`
+ -
+ - m2unet_covd-drive.pth_
+ * - :py:mod:`drive-ssl <bob.ip.binseg.configs.datasets.drive.ssl>`
+ -
+ - m2unet_covd-drive_ssl.pth_
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
+ - driu_stare.pth_
+ - m2unet_stare.pth_
+ * - :py:mod:`stare-covd <bob.ip.binseg.configs.datasets.stare.covd>`
+ -
+ - m2unet_covd-stare.pth_
+ * - :py:mod:`stare-ssl <bob.ip.binseg.configs.datasets.stare.ssl>`
+ -
+ - m2unet_covd-stare_ssl.pth_
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
+ - driu_chasedb1.pth_
+ - m2unet_chasedb1.pth_
+ * - :py:mod:`chasedb1-covd <bob.ip.binseg.configs.datasets.chasedb1.covd>`
+ -
+ - m2unet_covd-chasedb1.pth_
+ * - :py:mod:`chasedb1-ssl <bob.ip.binseg.configs.datasets.chasedb1.ssl>`
+ -
+ - m2unet_covd-chasedb1_ssl.pth_
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
+ - driu_iostar.pth_
+ - m2unet_iostar.pth_
+ * - :py:mod:`iostar-vessel-covd <bob.ip.binseg.configs.datasets.iostar.covd>`
+ -
+ - m2unet_covd-iostar.pth_
+ * - :py:mod:`iostar-vessel-ssl <bob.ip.binseg.configs.datasets.iostar.ssl>`
+ -
+ - m2unet_covd-iostar_ssl.pth_
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`
+ - driu_hrf.pth_
+ - m2unet_hrf.pth_
+ * - :py:mod:`hrf-covd <bob.ip.binseg.configs.datasets.hrf.covd>`
+ -
+ - m2unet_covd-hrf.pth_
+ * - :py:mod:`hrf-ssl <bob.ip.binseg.configs.datasets.hrf.ssl>`
+ -
+ - m2unet_covd-hrf_ssl.pth_
+
+.. include:: links.rst
diff --git a/doc/nitpick-exceptions.txt b/doc/nitpick-exceptions.txt
index bd53da1a83ba588f4c110926093fc88ba0016501..8f6fe3b338c959f4f3113d1280c57228351a8c0a 100644
--- a/doc/nitpick-exceptions.txt
+++ b/doc/nitpick-exceptions.txt
@@ -1,6 +1,2 @@
-py:class torch.nn.modules.module.Module
py:class torch.nn.modules.loss._Loss
-py:class torch.utils.data.dataset.Dataset
py:class Module
-py:mod bob.db.base
-py:obj list
diff --git a/doc/plotting.rst b/doc/plotting.rst
deleted file mode 100644
index f05ee42d12572211ffe4037a1c7d706283042a63..0000000000000000000000000000000000000000
--- a/doc/plotting.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.plotting:
-
-========
-Plotting
-========
-
-Precision vs recall curves for each evaluation run are generated by default and
-stored in the ``results`` subfolder of the model output directory.
-
-To generate a comparison chart of various models use the ``compare`` command
-and pass as arguments the output paths of the models you would like to plot.
-
-E.g.:
-
-.. code-block:: bash
-
- bob binseg compare -o myoutput -l myoutput/DRIVE/M2U-Net myoutput/DRIVE/U-Net myoutput/DRIVE/HED -t MyPlotTitle
-
-Use ``bob binseg compare --help`` for more information.
diff --git a/doc/references.rst b/doc/references.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6b942813843865e8d8e9cc87d15952c3a3ad3af4
--- /dev/null
+++ b/doc/references.rst
@@ -0,0 +1,106 @@
+.. coding=utf-8
+
+============
+ References
+============
+
+.. [STARE-2000] *A. D. Hoover, V. Kouznetsova and M. Goldbaum*, **Locating blood
+ vessels in retinal images by piecewise threshold probing of a matched filter
+ response**, in IEEE Transactions on Medical Imaging, vol. 19, no. 3, pp.
+ 203-210, March 2000. https://doi.org/10.1109/42.845178
+
+.. [DRIVE-2004] *J. Staal, M. D. Abramoff, M. Niemeijer, M. A. Viergever and B.
+ van Ginneken*, **Ridge-based vessel segmentation in color images of the
+ retina**, in IEEE Transactions on Medical Imaging, vol. 23, no. 4, pp.
+ 501-509, April 2004. https://doi.org/10.1109/TMI.2004.825627
+
+.. [CHASEDB1-2012] *M. M. Fraz et al.*, **An Ensemble Classification-Based
+ Approach Applied to Retinal Blood Vessel Segmentation**, in IEEE
+ Transactions on Biomedical Engineering, vol. 59, no. 9, pp. 2538-2548, Sept.
+ 2012. https://doi.org/10.1109/TBME.2012.2205687
+
+.. [HRF-2013] *A. Budai, R. Bock, A. Maier, J. Hornegger, and G. Michelson*,
+ **Robust Vessel Segmentation in Fundus Images**, in International Journal of
+ Biomedical Imaging, vol. 2013, p. 11, 2013.
+ http://dx.doi.org/10.1155/2013/154860
+
+.. [IOSTAR-2016] *J. Zhang, B. Dashtbozorg, E. Bekkers, J. P. W. Pluim, R. Duits
+ and B. M. ter Haar Romeny*, **Robust Retinal Vessel Segmentation via Locally
+ Adaptive Derivative Frames in Orientation Scores**, in IEEE Transactions on
+ Medical Imaging, vol. 35, no. 12, pp. 2631-2644, Dec. 2016.
+ https://doi.org/10.1109/TMI.2016.2587062
+
+.. [DRIONSDB-2008] *Enrique J. Carmona, Mariano Rincón, Julián GarcÃa-Feijoó, José
+ M. MartÃnez-de-la-Casa*, **Identification of the optic nerve head with
+ genetic algorithms**, in Artificial Intelligence in Medicine, Volume 43,
+ Issue 3, pp. 243-259, 2008. http://dx.doi.org/10.1016/j.artmed.2008.04.005
+
+.. [RIMONER3-2015] *F. Fumero, J. Sigut, S. AlayoÌn, M. GonzaÌlez-HernaÌndez, M.
+ GonzaÌlez de la Rosa*, **Interactive Tool and Database for Optic Disc and Cup
+ Segmentation of Stereo and Monocular Retinal Fundus Images**, Conference on
+ Computer Graphics, Visualization and Computer Vision, 2015.
+ https://dspace5.zcu.cz/bitstream/11025/29670/1/Fumero.pdf
+
+.. [DRISHTIGS1-2014] *J. Sivaswamy, S. R. Krishnadas, G. Datt Joshi, M. Jain and
+ A. U. Syed Tabish*, **Drishti-GS: Retinal image dataset for optic nerve
+ head (ONH) segmentation**, 2014 IEEE 11th International Symposium on
+ Biomedical Imaging (ISBI), Beijing, 2014, pp. 53-56.
+ https://doi.org/10.1109/ISBI.2014.6867807
+
+.. [REFUGE-2018] https://refuge.grand-challenge.org/Details/
+
+.. [MANINIS-2016] *K.-K. Maninis, J. Pont-Tuset, P. Arbeláez, and L. Van Gool*,
+ **Deep Retinal Image Understanding**, in Medical Image Computing and
+ Computer-Assisted Intervention – MICCAI 2016, Cham, 2016, pp. 140–148.
+ https://doi.org/10.1007/978-3-319-46723-8_17
+
+.. [ORLANDO-2017] *J. I. Orlando, E. Prokofyeva and M. B. Blaschko*, **A
+ Discriminatively Trained Fully Connected Conditional Random Field Model for
+ Blood Vessel Segmentation in Fundus Images**, in IEEE Transactions on
+ Biomedical Engineering, vol. 64, no. 1, pp. 16-27, Jan. 2017.
+ https://doi.org/10.1109/TBME.2016.2535311
+
+.. [MEYER-2017] *M. I. Meyer, P. Costa, A. Galdran, A. M. Mendonça, and A.
+ Campilho*, **A Deep Neural Network for Vessel Segmentation of Scanning Laser
+ Ophthalmoscopy Images**, in Image Analysis and Recognition, vol. 10317, F.
+ Karray, A. Campilho, and F. Cheriet, Eds. Cham: Springer International
+ Publishing, 2017, pp. 507–515. https://doi.org/10.1007/978-3-319-59876-5_56
+
+.. [IGLOVIKOV-2018] *V. Iglovikov, S. Seferbekov, A. Buslaev and A. Shvets*,
+ **TernausNetV2: Fully Convolutional Network for Instance Segmentation**,
+ 2018 IEEE/CVF Conference on Computer Vision and Pattern Recognition
+ Workshops (CVPRW), Salt Lake City, UT, 2018, pp. 228-2284.
+ https://doi.org/10.1109/CVPRW.2018.00042
+
+.. [HE-2015] *S. Xie and Z. Tu*, **Holistically-Nested Edge Detection**, 2015
+ IEEE International Conference on Computer Vision (ICCV), Santiago, 2015, pp.
+ 1395-1403. https://doi.org/10.1109/ICCV.2015.164
+
+.. [LUO-2019] *L. Luo, Y. Xiong, Y. Liu, and X. Sun*, **Adaptive Gradient
+ Methods with Dynamic Bound of Learning Rate**, Proceedings of the 7th
+ International Conference on Learning Representations (ICLR), Feb. 2019.
+ https://arxiv.org/abs/1902.09843v1
+
+.. [MASSA-2018] *F. Massa and R. Girshick*, **maskrcnn-benchmark: Fast, modular
+ reference implementation of Instance Segmentation and Object Detection
+ algorithms in PyTorch**. 2018. Last accessed: 21.03.2020.
+ https://github.com/facebookresearch/maskrcnn-benchmark
+
+.. [LIN-2018] *J. Lin*, **pytorch-mobilenet-v2: A PyTorch implementation of
+ MobileNetV2**, 2018. Last accessed: 21.03.2020.
+ https://github.com/tonylins/pytorch-mobilenet-v2
+
+.. [XIE-2015] *S. Xie and Z. Tu*, **Holistically-Nested Edge Detection**, 2015
+ IEEE International Conference on Computer Vision (ICCV), Santiago, 2015, pp.
+ 1395-1403. https://doi.org/10.1109/ICCV.2015.164
+
+.. [RONNEBERGER-2015] *O. Ronneberger, P. Fischer, T. Brox*, **U-Net:
+ Convolutional Networks for Biomedical Image Segmentation**, 2015.
+ https://arxiv.org/abs/1505.04597
+
+.. [ZHANG-2017] *Z. Zhang, Q. Liu, Y. Wang*, **Road Extraction by Deep Residual
+ U-Net**, 2017. https://arxiv.org/abs/1711.10684
+
+.. [SANDLER-2018] *M. Sandler, A. Howard, M. Zhu, A. Zhmoginov, L.-C.h Chen*,
+ **MobileNetV2: Inverted Residuals and Linear Bottlenecks**, 2018.
+ https://arxiv.org/abs/1801.04381
diff --git a/doc/results/baselines/chasedb1.pdf b/doc/results/baselines/chasedb1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..3139798feaa762bc1c9a11e8a6716a5ae21e9a63
Binary files /dev/null and b/doc/results/baselines/chasedb1.pdf differ
diff --git a/doc/results/baselines/chasedb1.png b/doc/results/baselines/chasedb1.png
new file mode 100644
index 0000000000000000000000000000000000000000..0f760b2a097c2b42fa0e067a5a9920d0a525788c
Binary files /dev/null and b/doc/results/baselines/chasedb1.png differ
diff --git a/doc/results/baselines/drive.pdf b/doc/results/baselines/drive.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..3fba78c86bf8c0f20d8ac2612b87a67cdad0b20b
Binary files /dev/null and b/doc/results/baselines/drive.pdf differ
diff --git a/doc/results/baselines/drive.png b/doc/results/baselines/drive.png
new file mode 100644
index 0000000000000000000000000000000000000000..9ecae7f663dde499a0cd3999e0aeaecf17aab778
Binary files /dev/null and b/doc/results/baselines/drive.png differ
diff --git a/doc/results/baselines/hrf.pdf b/doc/results/baselines/hrf.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..03bd6092d55df80b9f35c04e23caec13828d80f3
Binary files /dev/null and b/doc/results/baselines/hrf.pdf differ
diff --git a/doc/results/baselines/hrf.png b/doc/results/baselines/hrf.png
new file mode 100644
index 0000000000000000000000000000000000000000..1608a3e7fdcab0bac4321b85945eb6766abb5827
Binary files /dev/null and b/doc/results/baselines/hrf.png differ
diff --git a/doc/results/baselines/index.rst b/doc/results/baselines/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ce5361fbe43cf1da75b49bcdf03a673f3efce830
--- /dev/null
+++ b/doc/results/baselines/index.rst
@@ -0,0 +1,136 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.baselines:
+
+===================
+ Baseline Results
+===================
+
+F1 Scores (micro-level)
+-----------------------
+
+* Benchmark results for models: DRIU, HED, M2U-Net and U-Net.
+* Models are trained and tested on the same dataset (**numbers in bold**
+ indicate number of parameters per model). Models are trained for a fixed
+ number of 1000 epochs, with a learning rate of 0.001 until epoch 900 and then
+ 0.0001 until the end of the training.
+* Database and model resource configuration links (table top row and left
+ column) are linked to the originating configuration files used to obtain
+ these results.
+* Check `our paper`_ for details on the calculation of the F1 Score and standard
+ deviations (in parentheses).
+* Single performance numbers correspond to *a priori* performance indicators,
+ where the threshold is previously selected on the training set
+* You can cross check the analysis numbers provided in this table by
+ downloading this software package, the raw data, and running ``bob binseg
+ analyze`` providing the model URL as ``--weight`` parameter.
+* For comparison purposes, we provide "second-annotator" performances on the
+ same test set, where available.
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ -
+ - :py:mod:`driu <bob.ip.binseg.configs.models.driu>`
+ - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
+ - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+ - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
+ * - Dataset
+ - 2nd. Annot.
+ - 15M
+ - 14.7M
+ - 0.55M
+ - 25.8M
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`
+ - 0.788 (0.021)
+ - `0.819 (0.016) <baselines_driu_drive_>`_
+ - `0.806 (0.015) <baselines_hed_drive_>`_
+ - `0.804 (0.014) <baselines_m2unet_drive_>`_
+ - `0.823 (0.015) <baselines_unet_drive_>`_
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`
+ - 0.759 (0.028)
+ - `0.824 (0.037) <baselines_driu_stare_>`_
+ - `0.810 (0.045) <baselines_hed_stare_>`_
+ - `0.811 (0.039) <baselines_m2unet_stare_>`_
+ - `0.828 (0.041) <baselines_unet_stare_>`_
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`
+ - 0.768 (0.023)
+ - `0.811 (0.018) <baselines_driu_chase_>`_
+ - `0.806 (0.021) <baselines_hed_chase_>`_
+ - `0.801 (0.018) <baselines_m2unet_chase_>`_
+ - `0.802 (0.015) <baselines_unet_chase_>`_
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`
+ -
+ - `0.802 (0.039) <baselines_driu_hrf_>`_
+ - `0.793 (0.041) <baselines_hed_hrf_>`_
+ - `0.796 (0.043) <baselines_m2unet_hrf_>`_
+ - `0.798 (0.038) <baselines_unet_hrf_>`_
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`
+ -
+ - `0.825 (0.021) <baselines_driu_iostar_>`_
+ - `0.822 (0.023) <baselines_hed_iostar_>`_
+ - `0.817 (0.021) <baselines_m2unet_iostar_>`_
+ - `0.818 (0.019) <baselines_unet_iostar_>`_
+
+
+Precision-Recall (PR) Curves
+----------------------------
+
+Next, you will find the PR plots showing confidence intervals, for the various
+models explored, on a per dataset arrangement. All curves correspond to test
+set performances. Single performance figures (F1-micro scores) correspond to
+its average value across all test set images, for a fixed threshold set to
+``0.5``.
+
+.. list-table::
+
+ * - .. figure:: drive.png
+ :align: center
+ :scale: 50%
+ :alt: Model comparisons for drive datasets
+
+ :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>`: PR curve and F1 scores at T=0.5 (:download:`pdf <drive.pdf>`)
+ - .. figure:: stare.png
+ :align: center
+ :scale: 50%
+ :alt: Model comparisons for stare datasets
+
+ :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>`: PR curve and F1 scores at T=0.5 (:download:`pdf <stare.pdf>`)
+ * - .. figure:: chasedb1.png
+ :align: center
+ :scale: 50%
+ :alt: Model comparisons for chasedb1 datasets
+
+ :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>`: PR curve and F1 scores at T=0.5 (:download:`pdf <chasedb1.pdf>`)
+ - .. figure:: hrf.png
+ :align: center
+ :scale: 50%
+ :alt: Model comparisons for hrf datasets
+
+ :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>`: PR curve and F1 scores at T=0.5 (:download:`pdf <hrf.pdf>`)
+ * - .. figure:: iostar-vessel.png
+ :align: center
+ :scale: 50%
+ :alt: Model comparisons for iostar-vessel datasets
+
+ :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>`: PR curve and F1 scores at T=0.5 (:download:`pdf <iostar-vessel.pdf>`)
+ -
+
+
+Remarks
+-------
+
+* There seems to be no clear winner as confidence intervals based on the
+ standard deviation overlap substantially between the different models, and
+ across different datasets.
+* There seems to be almost no effect on the number of parameters on
+ performance. U-Net, the largest model, is not a clear winner through all
+ baseline benchmarks
+* Where second annotator labels exist, model performance and variability seems
+ on par with such annotations. One possible exception is for CHASE-DB1, where
+ models show consistently less variability than the second annotator.
+ Unfortunately, this cannot be conclusive.
+
+.. include:: ../../links.rst
diff --git a/doc/results/baselines/iostar-vessel.pdf b/doc/results/baselines/iostar-vessel.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..141ea565273373311ca3960cdb994ba2f2c6f58c
Binary files /dev/null and b/doc/results/baselines/iostar-vessel.pdf differ
diff --git a/doc/results/baselines/iostar-vessel.png b/doc/results/baselines/iostar-vessel.png
new file mode 100644
index 0000000000000000000000000000000000000000..7da4802492e1eac053166080e5b19090c2499f59
Binary files /dev/null and b/doc/results/baselines/iostar-vessel.png differ
diff --git a/doc/results/baselines/stare.pdf b/doc/results/baselines/stare.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..8bafd654a22dac56458355e048ede26ce4ce7cd1
Binary files /dev/null and b/doc/results/baselines/stare.pdf differ
diff --git a/doc/results/baselines/stare.png b/doc/results/baselines/stare.png
new file mode 100644
index 0000000000000000000000000000000000000000..7a632af2907d4c0f4d48d23072dcc97c189cf83f
Binary files /dev/null and b/doc/results/baselines/stare.png differ
diff --git a/doc/results/covd/index.rst b/doc/results/covd/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..844d9b963871fd3afcf2495d260d6778a758b3f6
--- /dev/null
+++ b/doc/results/covd/index.rst
@@ -0,0 +1,122 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.covd:
+
+.. todo::
+
+ This section is outdated and needs re-factoring.
+
+
+============================
+ COVD- and COVD-SLL Results
+============================
+
+In addition to the M2U-Net architecture, we also evaluated the larger DRIU
+network and a variation of it that contains batch normalization (DRIU+BN) on
+COVD- (Combined Vessel Dataset from all training data minus target test set)
+and COVD-SSL (COVD- and Semi-Supervised Learning). Perhaps surprisingly, for
+the majority of combinations, the performance of the DRIU variants are roughly
+equal or worse to the ones obtained with the much smaller M2U-Net. We
+anticipate that one reason for this could be overparameterization of large
+VGG-16 models that are pretrained on ImageNet.
+
+
+F1 Scores
+---------
+
+Comparison of F1 Scores (micro-level and standard deviation) of DRIU and
+M2U-Net on COVD- and COVD-SSL. Standard deviation across test-images in
+brackets.
+
+.. list-table::
+ :header-rows: 1
+
+ * - F1 score
+ - :py:mod:`DRIU <bob.ip.binseg.configs.models.driu>`/:py:mod:`DRIU@SSL <bob.ip.binseg.configs.models.driu_ssl>`
+ - :py:mod:`DRIU+BN <bob.ip.binseg.configs.models.driu_bn>`/:py:mod:`DRIU+BN@SSL <bob.ip.binseg.configs.models.driu_bn_ssl>`
+ - :py:mod:`M2U-Net <bob.ip.binseg.configs.models.m2unet>`/:py:mod:`M2U-Net@SSL <bob.ip.binseg.configs.models.m2unet_ssl>`
+ * - :py:mod:`COVD-DRIVE <bob.ip.binseg.configs.datasets.drive.covd>`
+ - 0.788 (0.018)
+ - 0.797 (0.019)
+ - `0.789 (0.018) <m2unet_covd-drive.pth>`_
+ * - :py:mod:`COVD-DRIVE+SSL <bob.ip.binseg.configs.datasets.drive.ssl>`
+ - 0.785 (0.018)
+ - 0.783 (0.019)
+ - `0.791 (0.014) <m2unet_covd-drive_ssl.pth>`_
+ * - :py:mod:`COVD-STARE <bob.ip.binseg.configs.datasets.stare.covd>`
+ - 0.778 (0.117)
+ - 0.778 (0.122)
+ - `0.812 (0.046) <m2unet_covd-stare.pth>`_
+ * - :py:mod:`COVD-STARE+SSL <bob.ip.binseg.configs.datasets.stare.ssl>`
+ - 0.788 (0.102)
+ - 0.811 (0.074)
+ - `0.820 (0.044) <m2unet_covd-stare_ssl.pth>`_
+ * - :py:mod:`COVD-CHASEDB1 <bob.ip.binseg.configs.datasets.chasedb1.covd>`
+ - 0.796 (0.027)
+ - 0.791 (0.025)
+ - `0.788 (0.024) <m2unet_covd-chasedb1.pth>`_
+ * - :py:mod:`COVD-CHASEDB1+SSL <bob.ip.binseg.configs.datasets.chasedb1.ssl>`
+ - 0.796 (0.024)
+ - 0.798 (0.025)
+ - `0.799 (0.026) <m2unet_covd-chasedb1_ssl.pth>`_
+ * - :py:mod:`COVD-HRF <bob.ip.binseg.configs.datasets.hrf.covd>`
+ - 0.799 (0.044)
+ - 0.800 (0.045)
+ - `0.802 (0.045) <m2unet_covd-hrf.pth>`_
+ * - :py:mod:`COVD-HRF+SSL <bob.ip.binseg.configs.datasets.hrf.ssl>`
+ - 0.799 (0.044)
+ - 0.784 (0.048)
+ - `0.797 (0.044) <m2unet_covd-hrf_ssl.pth>`_
+ * - :py:mod:`COVD-IOSTAR-VESSEL <bob.ip.binseg.configs.datasets.iostar.covd>`
+ - 0.791 (0.021)
+ - 0.777 (0.032)
+ - `0.793 (0.015) <m2unet_covd-iostar.pth>`_
+ * - :py:mod:`COVD-IOSTAR-VESSEL+SSL <bob.ip.binseg.configs.datasets.iostar.ssl>`
+ - 0.797 (0.017)
+ - 0.811 (0.074)
+ - `0.785 (0.018) <m2unet_covd-iostar_ssl.pth>`_
+
+
+M2U-Net Precision vs. Recall Curves
+-----------------------------------
+
+Precision vs. recall curves for each evaluated dataset. Note that here the
+F1-score is calculated on a macro level (see paper for more details).
+
+.. figure:: pr_CHASEDB1.png
+ :scale: 50 %
+ :align: center
+ :alt: model comparisons
+
+ CHASE_DB1: Precision vs Recall curve and F1 scores
+
+.. figure:: pr_DRIVE.png
+ :scale: 50 %
+ :align: center
+ :alt: model comparisons
+
+ DRIVE: Precision vs Recall curve and F1 scores
+
+.. figure:: pr_HRF.png
+ :scale: 50 %
+ :align: center
+ :alt: model comparisons
+
+ HRF: Precision vs Recall curve and F1 scores
+
+.. figure:: pr_IOSTARVESSEL.png
+ :scale: 50 %
+ :align: center
+ :alt: model comparisons
+
+ IOSTAR: Precision vs Recall curve and F1 scores
+
+.. figure:: pr_STARE.png
+ :scale: 50 %
+ :align: center
+ :alt: model comparisons
+
+ STARE: Precision vs Recall curve and F1 scores
+
+
+.. include:: ../../links.rst
diff --git a/doc/img/pr_CHASEDB1.png b/doc/results/covd/pr_CHASEDB1.png
similarity index 100%
rename from doc/img/pr_CHASEDB1.png
rename to doc/results/covd/pr_CHASEDB1.png
diff --git a/doc/img/pr_DRIVE.png b/doc/results/covd/pr_DRIVE.png
similarity index 100%
rename from doc/img/pr_DRIVE.png
rename to doc/results/covd/pr_DRIVE.png
diff --git a/doc/img/pr_HRF.png b/doc/results/covd/pr_HRF.png
similarity index 100%
rename from doc/img/pr_HRF.png
rename to doc/results/covd/pr_HRF.png
diff --git a/doc/img/pr_IOSTARVESSEL.png b/doc/results/covd/pr_IOSTARVESSEL.png
similarity index 100%
rename from doc/img/pr_IOSTARVESSEL.png
rename to doc/results/covd/pr_IOSTARVESSEL.png
diff --git a/doc/img/pr_STARE.png b/doc/results/covd/pr_STARE.png
similarity index 100%
rename from doc/img/pr_STARE.png
rename to doc/results/covd/pr_STARE.png
diff --git a/doc/results/index.rst b/doc/results/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..f2d7e2ac8de271a3745b1de154d0b848ef70806b
--- /dev/null
+++ b/doc/results/index.rst
@@ -0,0 +1,23 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results:
+
+=========
+ Results
+=========
+
+This section summarizes results that can be obtained with this package, and
+were presented in our paper. We organize the result section in two parts, for
+covering baseline results (training and testing on the same dataset) and
+results using our Combined Vessel Dataset minus target dataset (COVD-) training
+strategy.
+
+.. toctree::
+ :maxdepth: 2
+
+ baselines/index
+ xtest/index
+ covd/index
+
+
+.. include:: ../links.rst
diff --git a/doc/results/xtest/driu-chasedb1.pdf b/doc/results/xtest/driu-chasedb1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..bb28aafb8479d2f07336a7dfdcdd16fa148f5117
Binary files /dev/null and b/doc/results/xtest/driu-chasedb1.pdf differ
diff --git a/doc/results/xtest/driu-chasedb1.png b/doc/results/xtest/driu-chasedb1.png
new file mode 100644
index 0000000000000000000000000000000000000000..be26e9f8e8b140aaca692c417892abb515a180f4
Binary files /dev/null and b/doc/results/xtest/driu-chasedb1.png differ
diff --git a/doc/results/xtest/driu-drive.pdf b/doc/results/xtest/driu-drive.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..1f9fc10b1ffa612f6f889666598620f1007ef03f
Binary files /dev/null and b/doc/results/xtest/driu-drive.pdf differ
diff --git a/doc/results/xtest/driu-drive.png b/doc/results/xtest/driu-drive.png
new file mode 100644
index 0000000000000000000000000000000000000000..fba68683bf3a43a19a0a99c2f2f63d3f5d219473
Binary files /dev/null and b/doc/results/xtest/driu-drive.png differ
diff --git a/doc/results/xtest/driu-hrf.pdf b/doc/results/xtest/driu-hrf.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..01d78c98e041d4db603bce19ad549811b3cc3a81
Binary files /dev/null and b/doc/results/xtest/driu-hrf.pdf differ
diff --git a/doc/results/xtest/driu-hrf.png b/doc/results/xtest/driu-hrf.png
new file mode 100644
index 0000000000000000000000000000000000000000..0cbd94c9f9c9ffa71f5f513cd8f997070bc979ee
Binary files /dev/null and b/doc/results/xtest/driu-hrf.png differ
diff --git a/doc/results/xtest/driu-iostar-vessel.pdf b/doc/results/xtest/driu-iostar-vessel.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..db822d0f5c53d1579b8c71f3e339eefe2519e582
Binary files /dev/null and b/doc/results/xtest/driu-iostar-vessel.pdf differ
diff --git a/doc/results/xtest/driu-iostar-vessel.png b/doc/results/xtest/driu-iostar-vessel.png
new file mode 100644
index 0000000000000000000000000000000000000000..5842c71646109aecb0456cb20f99783b3b3bda0d
Binary files /dev/null and b/doc/results/xtest/driu-iostar-vessel.png differ
diff --git a/doc/results/xtest/driu-stare.pdf b/doc/results/xtest/driu-stare.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f44d12dafca83b3e2d3af52a48244ad1bd43365c
Binary files /dev/null and b/doc/results/xtest/driu-stare.pdf differ
diff --git a/doc/results/xtest/driu-stare.png b/doc/results/xtest/driu-stare.png
new file mode 100644
index 0000000000000000000000000000000000000000..6573b820be3e2402e732c45d1c106f0abb103aef
Binary files /dev/null and b/doc/results/xtest/driu-stare.png differ
diff --git a/doc/results/xtest/index.rst b/doc/results/xtest/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e65c9e9997e8dca62e0a6fcfd0bea45bae11906d
--- /dev/null
+++ b/doc/results/xtest/index.rst
@@ -0,0 +1,241 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.results.xtest:
+
+==========================
+ Cross-Database (X-)Tests
+==========================
+
+F1 Scores (micro-level)
+-----------------------
+
+* Models are trained and tested on the same dataset (numbers in parenthesis
+ indicate number of parameters per model), and then evaluated across the test
+ sets of other databases. X-tested datasets therefore represent *unseen*
+ data and can be a good proxy for generalisation analysis.
+* Each table row indicates a base trained model and each column the databases
+ the model was tested against. The native performance (intra-database) is
+ marked **in bold**. Thresholds are chosen *a priori* on the training set of
+ the database used to generate the model being cross-tested. Hence, the
+ threshold used for all experiments in a same row is always the same.
+* You can cross check the analysis numbers provided in this table by
+ downloading this software package, the raw data, and running ``bob binseg
+ analyze`` providing the model URL as ``--weight`` parameter, and then the
+ ``-xtest`` resource variant of the dataset the model was trained on. For
+ example, to run cross-evaluation tests for the DRIVE dataset, use the
+ configuration resource :py:mod:`drive-xtest
+ <bob.ip.binseg.configs.datasets.drive.xtest>`.
+* We only show results for DRIU (~15.4 million parameters) and M2U-Net (~550
+ thousand parameters) as these models seem to represent the performance
+ extremes according to our :ref:`baseline analysis
+ <bob.ip.binseg.results.baselines>`. You may run analysis on the other models
+ by downloading them from our website (via the ``--weight`` parameter on the
+ :ref:`analyze script <bob.ip.binseg.cli.analyze>`). This script may help you
+ in this task, provided you created a directory structure as suggested by
+ :ref:`our baseline script <bob.ip.binseg.baseline-script>`:
+
+ .. literalinclude:: ../../scripts/xtest.sh
+ :language: bash
+
+
+DRIU
+====
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ - drive
+ - stare
+ - chasedb1
+ - hrf
+ - iostar-vessel
+ * - Model / W x H
+ - 544 x 544
+ - 704 x 608
+ - 960 x 960
+ - 1648 x 1168
+ - 1024 x 1024
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_driu_drive_>`_)
+ - **0.819 (0.016)**
+ - 0.759 (0.151)
+ - 0.321 (0.068)
+ - 0.711 (0.067)
+ - 0.493 (0.049)
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_driu_stare_>`_)
+ - 0.733 (0.037)
+ - **0.824 (0.037)**
+ - 0.491 (0.094)
+ - 0.773 (0.051)
+ - 0.469 (0.055)
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_driu_chase_>`_)
+ - 0.730 (0.023)
+ - 0.730 (0.101)
+ - **0.811 (0.018)**
+ - 0.779 (0.043)
+ - 0.774 (0.019)
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_driu_hrf_>`_)
+ - 0.702 (0.038)
+ - 0.641 (0.160)
+ - 0.600 (0.072)
+ - **0.802 (0.039)**
+ - 0.546 (0.078)
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_driu_iostar_>`_)
+ - 0.758 (0.019)
+ - 0.724 (0.115)
+ - 0.777 (0.032)
+ - 0.727 (0.059)
+ - **0.825 (0.021)**
+
+
+Next, you will find the PR plots showing confidence intervals, for the various
+cross-tests explored, on a per cross-tested model arrangement. All curves
+correspond to test set performances. Single performance figures (F1-micro
+scores) correspond to its average value across all test set images, for a fixed
+threshold set *a priori* on the training set of dataset used for creating the
+model.
+
+.. list-table::
+
+ * - .. figure:: driu-drive.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a DRIU model based on DRIVE
+
+ :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.xtest>`: DRIU model X-tested (:download:`pdf <driu-drive.pdf>`)
+ - .. figure:: driu-stare.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a DRIU model based on STARE
+
+ :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.xtest>`: DRIU model X-tested (:download:`pdf <driu-stare.pdf>`)
+ * - .. figure:: driu-chasedb1.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a DRIU model based on CHASE-DB1
+
+ :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.xtest>`: DRIU model X-tested (:download:`pdf <driu-chasedb1.pdf>`)
+ - .. figure:: driu-hrf.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a DRIU model based on HRF
+
+ :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.xtest>`: DRIU model X-tested (:download:`pdf <driu-hrf.pdf>`)
+ * - .. figure:: driu-iostar-vessel.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a DRIU model based on IOSTAR (vessel)
+
+ :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel_xtest>`: DRIU model X-tested (:download:`pdf <driu-iostar-vessel.pdf>`)
+ -
+
+
+M2U-Net
+=======
+
+
+.. list-table::
+ :header-rows: 2
+
+ * -
+ - drive
+ - stare
+ - chasedb1
+ - hrf
+ - iostar-vessel
+ * - Model / W x H
+ - 544 x 544
+ - 704 x 608
+ - 960 x 960
+ - 1648 x 1168
+ - 1024 x 1024
+ * - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.default>` (`model <baselines_m2unet_drive_>`_)
+ - **0.804 (0.014)**
+ - 0.736 (0.144)
+ - 0.548 (0.055)
+ - 0.744 (0.058)
+ - 0.722 (0.036)
+ * - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.ah>` (`model <baselines_m2unet_stare_>`_)
+ - 0.715 (0.031)
+ - **0.811 (0.039)**
+ - 0.632 (0.033)
+ - 0.765 (0.049)
+ - 0.673 (0.033)
+ * - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.first_annotator>` (`model <baselines_m2unet_chase_>`_)
+ - 0.677 (0.027)
+ - 0.695 (0.099)
+ - **0.801 (0.018)**
+ - 0.763 (0.040)
+ - 0.761 (0.018)
+ * - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.default>` (`model <baselines_m2unet_hrf_>`_)
+ - 0.591 (0.071)
+ - 0.460 (0.230)
+ - 0.332 (0.108)
+ - **0.796 (0.043)**
+ - 0.419 (0.088)
+ * - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel>` (`model <baselines_m2unet_iostar_>`_)
+ - 0.743 (0.019)
+ - 0.745 (0.076)
+ - 0.771 (0.030)
+ - 0.749 (0.052)
+ - **0.817 (0.021)**
+
+
+Next, you will find the PR plots showing confidence intervals, for the various
+cross-tests explored, on a per cross-tested model arrangement. All curves
+correspond to test set performances. Single performance figures (F1-micro
+scores) correspond to its average value across all test set images, for a fixed
+threshold set *a priori* on the training set of dataset used for creating the
+model.
+
+.. list-table::
+
+ * - .. figure:: m2unet-drive.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a M2U-Net model based on DRIVE
+
+ :py:mod:`drive <bob.ip.binseg.configs.datasets.drive.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-drive.pdf>`)
+ - .. figure:: m2unet-stare.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a M2U-Net model based on STARE
+
+ :py:mod:`stare <bob.ip.binseg.configs.datasets.stare.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-stare.pdf>`)
+ * - .. figure:: m2unet-chasedb1.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a M2U-Net model based on CHASE-DB1
+
+ :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-chasedb1.pdf>`)
+ - .. figure:: m2unet-hrf.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a M2U-Net model based on HRF
+
+ :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf.xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-hrf.pdf>`)
+ * - .. figure:: m2unet-iostar-vessel.png
+ :align: center
+ :scale: 40%
+ :alt: X-tests for a M2U-Net model based on IOSTAR (vessel)
+
+ :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostar.vessel_xtest>`: M2U-Net model X-tested (:download:`pdf <m2unet-iostar-vessel.pdf>`)
+ -
+
+
+
+Remarks
+-------
+
+* For each row, the peak performance is always obtained in an intra-database
+ test (training and testing on the same database). Conversely, we observe a
+ performance degradation (albeit not catastrophic in most cases) for all other
+ datasets in the cross test.
+* X-test performance on a model created from HRF suggests a strong bias, as
+ performance does not generalize well for other (unseen) datasets.
+* Models generated from CHASE-DB1 and IOSTAR (vessel) seem to generalize quite
+ well to unseen data, when compared to the relatively poor generalization
+ capabilites of models generated from HRF or DRIVE.
+
+.. include:: ../../links.rst
diff --git a/doc/results/xtest/m2unet-chasedb1.pdf b/doc/results/xtest/m2unet-chasedb1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..22368ff4c89b9968a63a2c937ba1945f1dc881ec
Binary files /dev/null and b/doc/results/xtest/m2unet-chasedb1.pdf differ
diff --git a/doc/results/xtest/m2unet-chasedb1.png b/doc/results/xtest/m2unet-chasedb1.png
new file mode 100644
index 0000000000000000000000000000000000000000..f7fbaffad64fd42012f2394e412b7f4183ba2f05
Binary files /dev/null and b/doc/results/xtest/m2unet-chasedb1.png differ
diff --git a/doc/results/xtest/m2unet-drive.pdf b/doc/results/xtest/m2unet-drive.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..e8090cecb4c178646e0a0bb51da5b3a89f0b1548
Binary files /dev/null and b/doc/results/xtest/m2unet-drive.pdf differ
diff --git a/doc/results/xtest/m2unet-drive.png b/doc/results/xtest/m2unet-drive.png
new file mode 100644
index 0000000000000000000000000000000000000000..0b628ddfab5d6dc678d5f665f4e4eb3c7edec1fd
Binary files /dev/null and b/doc/results/xtest/m2unet-drive.png differ
diff --git a/doc/results/xtest/m2unet-hrf.pdf b/doc/results/xtest/m2unet-hrf.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..73d400cf279864de7fb3d372824606c2cdba79aa
Binary files /dev/null and b/doc/results/xtest/m2unet-hrf.pdf differ
diff --git a/doc/results/xtest/m2unet-hrf.png b/doc/results/xtest/m2unet-hrf.png
new file mode 100644
index 0000000000000000000000000000000000000000..ab4bcb45f2fa74bd6fa3b6a575723d71160a5c32
Binary files /dev/null and b/doc/results/xtest/m2unet-hrf.png differ
diff --git a/doc/results/xtest/m2unet-iostar-vessel.pdf b/doc/results/xtest/m2unet-iostar-vessel.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..6a59bc6ea23d7d17d54d08db8c8735b54445aae0
Binary files /dev/null and b/doc/results/xtest/m2unet-iostar-vessel.pdf differ
diff --git a/doc/results/xtest/m2unet-iostar-vessel.png b/doc/results/xtest/m2unet-iostar-vessel.png
new file mode 100644
index 0000000000000000000000000000000000000000..df9cc400f92f759f752161788e28a31826c17c94
Binary files /dev/null and b/doc/results/xtest/m2unet-iostar-vessel.png differ
diff --git a/doc/results/xtest/m2unet-stare.pdf b/doc/results/xtest/m2unet-stare.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..127f8d2abbc8aafd33fcbdcd4cc9613bea15b3b0
Binary files /dev/null and b/doc/results/xtest/m2unet-stare.pdf differ
diff --git a/doc/results/xtest/m2unet-stare.png b/doc/results/xtest/m2unet-stare.png
new file mode 100644
index 0000000000000000000000000000000000000000..e80cd25d1bce4604f62358d7d01bdfb1d4f67c6d
Binary files /dev/null and b/doc/results/xtest/m2unet-stare.png differ
diff --git a/doc/scripts/baselines.sh b/doc/scripts/baselines.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6f82b1aff351acd0a217a5e84221110483a75b4b
--- /dev/null
+++ b/doc/scripts/baselines.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Runs all of our baselines
+
+# set output directory and location of "bob" executable
+OUTDIR=/path/to/output/diretory
+BOB=/path/to/bob/execuble
+
+# run <modelconfig> <dbconfig> <batchsize> [<device> [<queue>]]
+function run() {
+ local device="cpu"
+ [ $# -gt 3 ] && device="${4}"
+
+ local cmd=(${BOB} binseg experiment)
+ cmd+=("-vv" "--device=${device}" ${1} ${2})
+ cmd+=("--batch-size=${3}" "--output-folder=${OUTDIR}/${1}/${2}")
+
+ # notice this assumes gridtk is installed
+ [ $# -gt 4 ] && cmd=(jman submit "--memory=24G" "--queue=${5}" -- "${cmd[@]}")
+
+ "${cmd[@]}"
+}
+
+# run/submit all baselines
+# comment out from "sgpu/gpu" to run locally
+# comment out from "cuda:0" to run on CPU
+run m2unet stare 6 #cuda:0 #sgpu
+run hed stare 4 #cuda:0 #sgpu
+run driu stare 5 #cuda:0 #sgpu
+run unet stare 2 #cuda:0 #sgpu
+run m2unet drive 16 #cuda:0 #sgpu
+run hed drive 8 #cuda:0 #sgpu
+run driu drive 8 #cuda:0 #sgpu
+run unet drive 4 #cuda:0 #sgpu
+run m2unet iostar-vessel 6 #cuda:0 #sgpu
+run hed iostar-vessel 4 #cuda:0 #sgpu
+run driu iostar-vessel 4 #cuda:0 #sgpu
+run unet iostar-vessel 2 #cuda:0 #sgpu
+run m2unet chasedb1 6 #cuda:0 #sgpu
+run hed chasedb1 4 #cuda:0 #sgpu
+run driu chasedb1 4 #cuda:0 #sgpu
+run unet chasedb1 2 #cuda:0 #sgpu
+run m2unet hrf 1 #cuda:0 #gpu
+run hed hrf 1 #cuda:0 #gpu
+run driu hrf 1 #cuda:0 #gpu
+run unet hrf 1 #cuda:0 #gpu
diff --git a/doc/scripts/xtest.sh b/doc/scripts/xtest.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6a198c98e5e3a53f57f39f25d17f1277c98a5977
--- /dev/null
+++ b/doc/scripts/xtest.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Runs cross database tests
+
+BOB=$HOME/work/bob/bob.ip.binseg/bin/bob
+
+for d in drive stare chasedb1 iostar-vessel hrf; do
+ for m in driu hed m2unet unet; do
+ cmd=(${BOB} binseg analyze -vv ${m} "${d}-xtest")
+ cmd+=("--weight=${m}/${d}/model/model_final.pth")
+ cmd+=("--output-folder=${m}/${d}/xtest")
+ "${cmd[@]}"
+ done
+done
diff --git a/doc/setup.rst b/doc/setup.rst
index 1200aac25c1074e9dc4695e0d78debdbcc995e5c..0955a905bc559e98a74106f1e6415425ebf7cc74 100644
--- a/doc/setup.rst
+++ b/doc/setup.rst
@@ -1,104 +1,72 @@
.. -*- coding: utf-8 -*-
.. _bob.ip.binseg.setup:
-=========
-Setup
-=========
+=======
+ Setup
+=======
-Bob.ip.binseg
-=============
+Complete Bob's `installation`_ instructions. Then, to install this package, do
+this:
-Complete bob's `installation`_ instructions. Then, to install this
-package
-
-.. code-block:: bash
-
- conda install bob.ip.binseg
-
-Datasets
-========
+.. code-block:: sh
-The package supports a range of retina fundus datasets but does not install the `bob.db`
-APIs by default, nor does it include the datasets.
+ $ conda activate <myenv>
+ (<myenv>) $ conda install bob.ip.binseg
-To setup a datasets:
+.. note::
-1. Download the dataset from the authors website
-2. Install the corresponding bob.db package via ``conda install bob.db.<database>``. E.g. to install the DRIVE API run ``conda install bob.db.drive``
-3. :ref:`datasetpathsetup`
-4. :ref:`dsconsistency`
+ The value ``<myenv>`` should correspond to the name of the environment where
+ you initially installed your Bob packages.
-+------------+----------------------------------------------------------------------+---------------------+
-| Dataset | Website | `bob.db` package |
-+------------+----------------------------------------------------------------------+---------------------+
-| STARE | http://cecas.clemson.edu/~ahoover/stare/ | `bob.db.stare` |
-+------------+----------------------------------------------------------------------+---------------------+
-| DRIVE | https://www.isi.uu.nl/Research/Databases/DRIVE/ | `bob.db.drive` |
-+------------+----------------------------------------------------------------------+---------------------+
-| DRIONS | http://www.ia.uned.es/~ejcarmona/DRIONS-DB.html | `bob.db.drionsdb` |
-+------------+----------------------------------------------------------------------+---------------------+
-| RIM-ONE | http://medimrg.webs.ull.es/research/downloads/ | `bob.db.rimoner3` |
-+------------+----------------------------------------------------------------------+---------------------+
-| CHASE-DB1 | https://blogs.kingston.ac.uk/retinal/chasedb1/ | `bob.db.chasedb` |
-+------------+----------------------------------------------------------------------+---------------------+
-| HRF | https://www5.cs.fau.de/research/data/fundus-images/ | `bob.db.hrf` |
-+------------+----------------------------------------------------------------------+---------------------+
-| Drishti-GS | http://cvit.iiit.ac.in/projects/mip/drishti-gs/mip-dataset2/Home.php | `bob.db.drishtigs1` |
-+------------+----------------------------------------------------------------------+---------------------+
-| IOSTAR | http://www.retinacheck.org/datasets | `bob.db.iostar` |
-+------------+----------------------------------------------------------------------+---------------------+
-| REFUGE | https://refuge.grand-challenge.org/Details/ | `bob.db.refuge` |
-+------------+----------------------------------------------------------------------+---------------------+
-.. _datasetpathsetup:
+Datasets
+--------
-Set up dataset paths
-=====================
+The package supports a range of retina fundus datasets, but does not include
+the raw data itself, which you must procure.
-For each dataset that you are planning to use, set the datadir to
-the path where it is stored. E.g.:
+To setup a dataset, do the following:
-.. code-block:: bash
+1. Download the dataset from the authors website (see
+ :ref:`bob.ip.binseg.datasets` for download links and details), unpack it and
+ store the directory leading to the uncompressed directory structure.
- bob config set bob.db.drive.datadir "/path/to/drivedataset/"
+ .. warning::
-To check your current setup
+ Our dataset connectors expect you provide "root" paths of raw datasets as
+ you unpack them in their **pristine** state. Changing the location of
+ files within a dataset distribution will likely cause execution errors.
-.. code-block:: bash
+2. For each dataset that you are planning to use, set the ``datadir`` to the
+ root path where it is stored. E.g.:
- bob config show
+ .. code-block:: sh
-This should result in an output similar to the following:
+ (<myenv>) $ bob config set bob.ip.binseg.drive.datadir "/path/to/drive"
-.. code-block:: bash
+ To check supported raw datasets and your current setup, do the following:
- {
- "bob.db.chasedb1.datadir": "/idiap/resource/database/CHASE-DB11/",
- "bob.db.drionsdb.datadir": "/idiap/resource/database/DRIONS",
- "bob.db.drishtigs1.datadir": "/idiap/resource/database/Drishti-GS1/",
- "bob.db.drive.datadir": "/idiap/resource/database/DRIVE",
- "bob.db.hrf.datadir": "/idiap/resource/database/HRF",
- "bob.db.iostar.datadir": "/idiap/resource/database/IOSTAR/IOSTAR Vessel Segmentation Dataset/",
- "bob.db.refuge.datadir": "/idiap/resource/database/REFUGE",
- "bob.db.rimoner3.datadir": "/idiap/resource/database/RIM-ONE/RIM-ONE r3",
- "bob.db.stare.datadir": "/idiap/resource/database/STARE"
- }
+ .. code-block:: sh
+ (<myenv>) $ bob binseg dataset list
+ Supported datasets:
+ - drive: bob.ip.binseg.drive.datadir = "/Users/andre/work/bob/dbs/drive"
+ * stare: bob.ip.binseg.stare.datadir (not set)
-.. _dsconsistency:
+ This command will show the set location for each configured dataset, and
+ the variable names for each supported dataset which has not yet been setup.
-Test dataset consitency
-========================
+3. To check whether the downloaded version is consistent with the structure
+ that is expected by this package, run ``bob binseg dataset check
+ <dataset>``, where ``<dataset>`` should be replaced by the
+ dataset programmatic name. E.g., to check DRIVE files, use:
-To check whether the downloaded version is consistent with
-the structure that is expected by our ``bob.db`` packages
-run ``bob_dbmanage.py datasettocheck checkfiles``
-E.g.:
+ .. code-block:: sh
-.. code-block:: sh
+ (<myenv>) $ bob binseg dataset check drive
+ ...
- conda activate your-conda-env-with-bob.ip.binseg
- bob_dbmanage.py drive checkfiles
- > checkfiles completed sucessfully
+ If there are problems on the current file organisation, this procedure
+ should detect and highlight which files are missing (cannot be loaded).
.. include:: links.rst
diff --git a/doc/training.rst b/doc/training.rst
index 5e5c83a4ada67722c274039429bd4731d844c72c..254cac14cfafcaf6f67698c2d41d62096c204d14 100644
--- a/doc/training.rst
+++ b/doc/training.rst
@@ -1,325 +1,21 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.training:
-
-
-========
-Training
-========
-
-To replicate our results use ``bob binseg train`` followed by the model config
-and the dataset config. Use ``bob binseg train --help`` for more information.
-
-.. note::
-
- We strongly advice training with a GPU (using ``-d cuda``). Depending on the available GPU
- memory you might have to adjust your batch size (``-b``).
-
-Default Dataset configs
-=======================
-
-1. Vessel:
-
-* CHASEDB1
-* CHASEDB1TEST
-* COVD-DRIVE
-* COVD-DRIVE_SSL
-* COVD-STARE
-* COVD-STARE_SSL
-* COVD-IOSTARVESSEL
-* COVD-IOSTARVESSEL_SSL
-* COVD-HRF
-* COVD-HRF_SSL
-* COVD-CHASEDB1
-* COVD-CHASEDB1_SSL
-* DRIVE
-* DRIVETEST
-* HRF
-* HRFTEST
-* IOSTARVESSEL
-* IOSTARVESSELTEST
-* STARE
-* STARETEST
-
-2. Optic Disc and Cup
-
-* DRIONSDB
-* DRIONSDBTEST
-* DRISHTIGS1OD
-* DRISHTIGS1ODTEST
-* DRISHTIGS1CUP
-* DRISHTIGS1CUPTEST
-* IOSTAROD
-* IOSTARODTEST
-* REFUGECUP
-* REFUGECUPTEST
-* REFUGEOD
-* REFUGEODTEST
-* RIMONER3CUP
-* RIMONER3CUPTEST
-* RIMONER3OD
-* RIMONER3ODTEST
-
-Default Model configs
-=====================
-
-* DRIU
-* DRIUBN
-* DRIUSSL
-* DRIUBNSSL
-* DRIUOD
-* HED
-* M2UNet
-* M2UNetSSL
-* UNet
-
-
-Baseline Benchmarks
-===================
-
-.. code-block:: bash
-
- #!/bin/bash
- # set output directory
- outputroot=`pwd`"/output"
- mkdir -p $outputroot
-
- #### Global config ####
- m2u=M2UNet
- hed=HED
- driu=DRIU
- unet=UNet
- m2ussl=M2UNetSSL
- driussl=DRIUSSL
-
- #### CHASE_DB 1 ####
- dataset=CHASEDB1
- output=$outputroot"/"$dataset
- mkdir -p $output
- # batch sizes
- b_m2u=6
- b_hed=4
- b_driu=4
- b_unet=2
- # Train
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
- bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
- #### DRIVE ####
- dataset=DRIVE
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- b_m2u=16
- b_hed=8
- b_driu=8
- b_unet=4
- # Train
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
- bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
- #### HRF ####
- dataset=HRF
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- b_m2u=1
- b_hed=1
- b_driu=1
- b_unet=1
- # Train
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
- bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
- #### IOSTAR VESSEL ####
- dataset=IOSTARVESSEL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- b_m2u=6
- b_hed=4
- b_driu=4
- b_unet=2
- # Train
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
- bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
- #### STARE ####
- dataset=STARE
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- b_m2u=6
- b_hed=4
- b_driu=5
- b_unet=2
- # Train
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
- bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
-
-Combined Vessel Dataset (COVD) and Semi-Supervised Learning (SSL)
-=================================================================
-
-COVD-:
-
-.. code-block:: bash
-
- ### COVD-DRIVE ####
- dataset=COVD-DRIVE
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIU
- m2u=M2UNet
- b_driu=4
- b_m2u=8
- # Train
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-STARE ####
- dataset=COVD-STARE
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIU
- m2u=M2UNet
- b_driu=4
- b_m2u=4
- # Train
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-IOSTAR ####
- dataset=COVD-IOSTARVESSEL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIU
- m2u=M2UNet
- b_driu=2
- b_m2u=4
- # Train
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-CHASEDB1 ####
- dataset=COVD-CHASEDB1
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIU
- m2u=M2UNet
- b_driu=2
- b_m2u=4
- # Train
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-HRF ####
- dataset=COVD-HRF
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIU
- m2u=M2UNet
- b_driu=2
- b_m2u=4
- # Train
- bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-
-COVD-SSL:
-
-.. code-block:: bash
-
- ### COVD-DRIVE_SSL ####
- dataset=COVD-DRIVE_SSL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIUSSL
- m2u=M2UNetSSL
- b_driu=4
- b_m2u=4
- # Train
- bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-STARE_SSL ####
- dataset=COVD-STARE_SSL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIUSSL
- m2u=M2UNetSSL
- b_driu=4
- b_m2u=4
- # Train
- bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-IOSTAR_SSL ####
- dataset=COVD-IOSTARVESSEL_SSL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIUSSL
- m2u=M2UNetSSL
- b_driu=1
- b_m2u=2
- # Train
- bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
- ### COVD-CHASEDB1_SSL ####
- dataset=COVD-CHASEDB1_SSL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIUSSL
- m2u=M2UNetSSL
- b_driu=2
- b_m2u=2
- # Train
- bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-
- ### COVD-HRF_SSL ####
- dataset=COVD-HRF_SSL
- output=$outputroot"/"$dataset
- mkdir -p $output
- # model configs
- driu=DRIUSSL
- m2u=M2UNetSSL
- b_driu=1
- b_m2u=2
- # Train
- bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
- bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-Using your own configs
-======================
+.. _bob.ip.binseg.training:
-Instead of the default configs you can pass the full path of your
-customized dataset and model config (both in PyTorch format).
-The default configs are stored under ``bob.ip.binseg/bob/ip/binseg/configs/``.
+==========
+ Training
+==========
-.. code-block:: bash
+To train a new FCN, use the command-line interface (CLI) application ``bob
+binseg train``, available on your prompt. To use this CLI, you must define the
+input dataset that will be used to train the FCN, as well as the type of model
+that will be trained. You may issue ``bob binseg train --help`` for a help
+message containing more detailed instructions.
- bob binseg train /path/to/model/config.py /path/to/dataset/config.py
+.. tip::
+ We strongly advice training with a GPU (using ``--device="cuda:0"``).
+ Depending on the available GPU memory you might have to adjust your batch
+ size (``--batch``).
diff --git a/doc/usage.rst b/doc/usage.rst
new file mode 100644
index 0000000000000000000000000000000000000000..b63cd4378782167b0c91b194167f30da1add4360
--- /dev/null
+++ b/doc/usage.rst
@@ -0,0 +1,68 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.usage:
+
+=======
+ Usage
+=======
+
+This package supports a fully reproducible research experimentation cycle for
+semantic binary segmentation with support for the following activities:
+
+* Training: Images are fed to a Fully Convolutional Deep Neural Network (FCN),
+ that is trained to reconstruct annotations (pre-segmented binary maps),
+ automatically, via error back propagation. The objective of this phase is to
+ produce an FCN model.
+* Inference (prediction): The FCN is used to generate vessel map predictions
+* Evaluation: Vessel map predictions are used evaluate FCN performance against
+ provided annotations, or visualize prediction results overlayed on
+ the original raw images.
+* Comparison: Use evaluation results to compare performance as you like.
+
+Whereas we provide :ref:`command-line interfaces (CLI)
+<bob.ip.binseg.cli.single>` that implement each of the phases above, we also
+provide command aggregators that can :ref:`run all of the phases
+<bob.ip.binseg.cli.combined>`. Both interfaces are configurable using
+:ref:`Bob's extensible configuration framework <bob.extension.framework>`. In
+essence, each command-line option may be provided as a variable with the same
+name in a Python file. Each file may combine any number of variables that are
+pertinent to an application.
+
+.. tip::
+
+ For reproducibility, we recommend you stick to configuration files when
+ parameterizing our CLI. Notice some of the options in the CLI interface
+ (e.g. ``--dataset``) cannot be passed via the actual command-line as it
+ may require complex Python types that cannot be synthetized in a single
+ input parameter.
+
+
+The following flowchart represents the various experiment phases and output
+results that can be produced for each of our CLI interfaces (rounded white
+rectangles). Processing subproducts (marked in blue), are stored on disk by
+the end of each step.
+
+.. graphviz:: framework.dot
+ :caption: Framework actions and CLI
+
+
+We provide a number of :ref:`preset configuration files
+<bob.ip.binseg.cli.config.list.all>` that can be used in one or more of the
+activities described in this section. Our command-line framework allows you to
+refer to these preset configuration files using special names (a.k.a.
+"resources"), that procure and load these for you automatically. Aside preset
+configuration files, you may also create your own to extend existing baseline
+experiments by :ref:`locally copying <bob.ip.binseg.cli.config.copy>` and
+modifying one of our configuration resources.
+
+
+.. toctree::
+ :maxdepth: 2
+
+ experiment
+ training
+ models
+ evaluation
+
+
+.. include:: links.rst
diff --git a/doc/visualization.rst b/doc/visualization.rst
deleted file mode 100644
index 56728e9562003c676dab0a5d51ca4640b12df1e8..0000000000000000000000000000000000000000
--- a/doc/visualization.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.visualization:
-
-=============
-Visualization
-=============
-
-Two visualization are generated via the ``bob binseg visualize`` command:
-
-1. Visualizations of true positives, false positives and false negatives
-overlayed over the test images
-2. Visualizations of the probability map outputs overlayed over the test images
-
-The following directory structure is expected:
-
-.. code-block:: bash
-
- ├── DATABASE
- ├── MODEL
- ├── images
- └── results
-
-Example to generate visualization for outputs for the DRIVE dataset:
-
-.. code-block:: bash
-
- # Visualizations are stored in the same output folder.
- bob binseg visualize DRIVETEST -o /DRIVE/M2UNet/output
-
-Use ``bob binseg visualize --help`` for more information.
diff --git a/requirements.txt b/requirements.txt
index 82b7843aa0e051bad5f22858752465620aca6772..706cd10addac90fafe7f8720b1e2d7aacde87445 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,12 @@
-setuptools
-numpy
bob.extension
+matplotlib
+numpy
+pandas
+h5py
+pillow
+psutil
+setuptools
+tabulate
torch
torchvision
-pandas
-matplotlib
tqdm
-tabulate
-bob.core
diff --git a/setup.py b/setup.py
index 8a7dd13d8cb3f4ce5e7730167eccbc9db5f49405..bff45cf36196001418a51894c65f90f82d3066e2 100644
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,6 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from setuptools import setup, dist
dist.Distribution(dict(setup_requires=["bob.extension"]))
-
from bob.extension.utils import load_requirements, find_packages
install_requires = load_requirements()
@@ -33,63 +29,94 @@ setup(
entry_points={
# main entry for bob binseg cli
"bob.cli": ["binseg = bob.ip.binseg.script.binseg:binseg"],
- # bob hed sub-commands
+ # bob binseg sub-commands
"bob.ip.binseg.cli": [
- "train = bob.ip.binseg.script.binseg:train",
- "test = bob.ip.binseg.script.binseg:test",
- "compare = bob.bin.binseg.script.binseg:compare",
- "gridtable = bob.ip.binseg.script.binseg:testcheckpoints",
- "visualize = bob.ip.binseg.script.binseg:visualize",
+ "config = bob.ip.binseg.script.config:config",
+ "dataset = bob.ip.binseg.script.dataset:dataset",
+ "train = bob.ip.binseg.script.train:train",
+ "predict = bob.ip.binseg.script.predict:predict",
+ "evaluate = bob.ip.binseg.script.evaluate:evaluate",
+ "compare = bob.ip.binseg.script.compare:compare",
+ "analyze = bob.ip.binseg.script.analyze:analyze",
+ "experiment = bob.ip.binseg.script.experiment:experiment",
],
# bob train configurations
"bob.ip.binseg.config": [
- "DRIU = bob.ip.binseg.configs.models.driu",
- "DRIUBN = bob.ip.binseg.configs.models.driubn",
- "DRIUSSL = bob.ip.binseg.configs.models.driussl",
- "DRIUBNSSL = bob.ip.binseg.configs.models.driubnssl",
- "DRIUOD = bob.ip.binseg.configs.models.driuod",
- "HED = bob.ip.binseg.configs.models.hed",
- "M2UNet = bob.ip.binseg.configs.models.m2unet",
- "M2UNetSSL = bob.ip.binseg.configs.models.m2unetssl",
- "UNet = bob.ip.binseg.configs.models.unet",
- "ResUNet = bob.ip.binseg.configs.models.resunet",
- "IMAGEFOLDER = bob.ip.binseg.configs.datasets.imagefolder",
- "CHASEDB1 = bob.ip.binseg.configs.datasets.chasedb1",
- "CHASEDB1TEST = bob.ip.binseg.configs.datasets.chasedb1test",
- "COVD-DRIVE = bob.ip.binseg.configs.datasets.starechasedb1iostarhrf544",
- "COVD-DRIVE_SSL = bob.ip.binseg.configs.datasets.starechasedb1iostarhrf544ssldrive",
- "COVD-STARE = bob.ip.binseg.configs.datasets.drivechasedb1iostarhrf608",
- "COVD-STARE_SSL = bob.ip.binseg.configs.datasets.drivechasedb1iostarhrf608sslstare",
- "COVD-IOSTARVESSEL = bob.ip.binseg.configs.datasets.drivestarechasedb1hrf1024",
- "COVD-IOSTARVESSEL_SSL = bob.ip.binseg.configs.datasets.drivestarechasedb1hrf1024ssliostar",
- "COVD-HRF = bob.ip.binseg.configs.datasets.drivestarechasedb1iostar1168",
- "COVD-HRF_SSL = bob.ip.binseg.configs.datasets.drivestarechasedb1iostar1168sslhrf",
- "COVD-CHASEDB1 = bob.ip.binseg.configs.datasets.drivestareiostarhrf960",
- "COVD-CHASEDB1_SSL = bob.ip.binseg.configs.datasets.drivestareiostarhrf960sslchase",
- "DRIONSDB = bob.ip.binseg.configs.datasets.drionsdb",
- "DRIONSDBTEST = bob.ip.binseg.configs.datasets.drionsdbtest",
- "DRISHTIGS1OD = bob.ip.binseg.configs.datasets.dristhigs1od",
- "DRISHTIGS1ODTEST = bob.ip.binseg.configs.datasets.dristhigs1odtest",
- "DRISHTIGS1CUP = bob.ip.binseg.configs.datasets.dristhigs1cup",
- "DRISHTIGS1CUPTEST = bob.ip.binseg.configs.datasets.dristhigs1cuptest",
- "DRIVE = bob.ip.binseg.configs.datasets.drive",
- "DRIVETEST = bob.ip.binseg.configs.datasets.drivetest",
- "HRF = bob.ip.binseg.configs.datasets.hrf1168",
- "HRFTEST = bob.ip.binseg.configs.datasets.hrftest",
- "IOSTAROD = bob.ip.binseg.configs.datasets.iostarod",
- "IOSTARODTEST = bob.ip.binseg.configs.datasets.iostarodtest",
- "IOSTARVESSEL = bob.ip.binseg.configs.datasets.iostarvessel",
- "IOSTARVESSELTEST = bob.ip.binseg.configs.datasets.iostarvesseltest",
- "REFUGECUP = bob.ip.binseg.configs.datasets.refugecup",
- "REFUGECUPTEST = bob.ip.binseg.configs.datasets.refugecuptest",
- "REFUGEOD = bob.ip.binseg.configs.datasets.refugeod",
- "REFUGEODTEST = bob.ip.binseg.configs.datasets.refugeodtest",
- "RIMONER3CUP = bob.ip.binseg.configs.datasets.rimoner3cup",
- "RIMONER3CUPTEST = bob.ip.binseg.configs.datasets.rimoner3cuptest",
- "RIMONER3OD = bob.ip.binseg.configs.datasets.rimoner3od",
- "RIMONER3ODTEST = bob.ip.binseg.configs.datasets.rimoner3odtest",
- "STARE = bob.ip.binseg.configs.datasets.stare",
- "STARETEST = bob.ip.binseg.configs.datasets.staretest",
+
+ # models
+ "driu = bob.ip.binseg.configs.models.driu",
+ "driu-bn = bob.ip.binseg.configs.models.driu_bn",
+ "driu-ssl = bob.ip.binseg.configs.models.driu_ssl",
+ "driu-bn-ssl = bob.ip.binseg.configs.models.driu_bn_ssl",
+ "driu-od = bob.ip.binseg.configs.models.driu_od",
+ "hed = bob.ip.binseg.configs.models.hed",
+ "m2unet = bob.ip.binseg.configs.models.m2unet",
+ "m2unet-ssl = bob.ip.binseg.configs.models.m2unet_ssl",
+ "unet = bob.ip.binseg.configs.models.unet",
+ "resunet = bob.ip.binseg.configs.models.resunet",
+
+ # example datasets
+ "csv-dataset-example = bob.ip.binseg.configs.datasets.csv",
+
+ # drive dataset
+ "drive = bob.ip.binseg.configs.datasets.drive.default",
+ "drive-2nd = bob.ip.binseg.configs.datasets.drive.second_annotator",
+ "drive-xtest = bob.ip.binseg.configs.datasets.drive.xtest",
+ "drive-mtest = bob.ip.binseg.configs.datasets.drive.mtest",
+ "drive-covd = bob.ip.binseg.configs.datasets.drive.covd",
+ "drive-ssl = bob.ip.binseg.configs.datasets.drive.ssl",
+
+ # stare dataset
+ "stare = bob.ip.binseg.configs.datasets.stare.ah",
+ "stare-2nd = bob.ip.binseg.configs.datasets.stare.vk",
+ "stare-xtest = bob.ip.binseg.configs.datasets.stare.xtest",
+ "stare-mtest = bob.ip.binseg.configs.datasets.stare.mtest",
+ "stare-covd = bob.ip.binseg.configs.datasets.stare.covd",
+ "stare-ssl = bob.ip.binseg.configs.datasets.stare.ssl",
+
+ # iostar
+ "iostar-vessel = bob.ip.binseg.configs.datasets.iostar.vessel",
+ "iostar-vessel-xtest = bob.ip.binseg.configs.datasets.iostar.vessel_xtest",
+ "iostar-vessel-mtest = bob.ip.binseg.configs.datasets.iostar.vessel_mtest",
+ "iostar-disc = bob.ip.binseg.configs.datasets.iostar.optic_disc",
+ "iostar-vessel-covd = bob.ip.binseg.configs.datasets.iostar.covd",
+ "iostar-vessel-ssl = bob.ip.binseg.configs.datasets.iostar.ssl",
+
+ # hrf
+ "hrf = bob.ip.binseg.configs.datasets.hrf.default",
+ "hrf-xtest = bob.ip.binseg.configs.datasets.hrf.xtest",
+ "hrf-mtest = bob.ip.binseg.configs.datasets.hrf.mtest",
+ "hrf-highres = bob.ip.binseg.configs.datasets.hrf.default_fullres",
+ "hrf-covd = bob.ip.binseg.configs.datasets.hrf.covd",
+ "hrf-ssl = bob.ip.binseg.configs.datasets.hrf.ssl",
+
+ # chase-db1
+ "chasedb1 = bob.ip.binseg.configs.datasets.chasedb1.first_annotator",
+ "chasedb1-2nd = bob.ip.binseg.configs.datasets.chasedb1.second_annotator",
+ "chasedb1-xtest = bob.ip.binseg.configs.datasets.chasedb1.xtest",
+ "chasedb1-mtest = bob.ip.binseg.configs.datasets.chasedb1.mtest",
+ "chasedb1-covd = bob.ip.binseg.configs.datasets.chasedb1.covd",
+ "chasedb1-ssl = bob.ip.binseg.configs.datasets.chasedb1.ssl",
+
+ # drionsdb
+ "drionsdb = bob.ip.binseg.configs.datasets.drionsdb.expert1",
+ "drionsdb-2nd = bob.ip.binseg.configs.datasets.drionsdb.expert2",
+
+ # drishti-gs1
+ "drishtigs1-disc = bob.ip.binseg.configs.datasets.drishtigs1.disc_all",
+ "drishtigs1-cup = bob.ip.binseg.configs.datasets.drishtigs1.cup_all",
+ "drishtigs1-disc-any = bob.ip.binseg.configs.datasets.drishtigs1.disc_any",
+ "drishtigs1-cup-any = bob.ip.binseg.configs.datasets.drishtigs1.cup_any",
+
+ # refuge
+ "refuge-cup = bob.ip.binseg.configs.datasets.refuge.cup",
+ "refuge-disc = bob.ip.binseg.configs.datasets.refuge.disc",
+
+ # rim one r3
+ "rimoner3-cup = bob.ip.binseg.configs.datasets.rimoner3.cup_exp1",
+ "rimoner3-disc = bob.ip.binseg.configs.datasets.rimoner3.disc_exp1",
+ "rimoner3-cup-2nd = bob.ip.binseg.configs.datasets.rimoner3.cup_exp2",
+ "rimoner3-disc-2nd = bob.ip.binseg.configs.datasets.rimoner3.disc_exp2",
],
},
# check classifiers, add and remove as you see fit
@@ -97,7 +124,7 @@ setup(
# don't remove the Bob framework unless it's not a bob package
classifiers=[
"Framework :: Bob",
- "Development Status :: 4 - Beta",
+ "Development Status :: 5 - Production/Stable",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Natural Language :: English",