diff --git a/doc/api.rst b/doc/api.rst index 1eade48273e9a1117484ac0d14a7df6b510b5e3d..2f732d29d89e121b2aaba42b6d90f6302d78b167 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -11,40 +11,57 @@ run binary-segmentation benchmarks. PyTorch bob.db Dataset ====================== + .. automodule:: bob.ip.binseg.data.binsegdataset + PyTorch ImageFolder Dataset =========================== + .. automodule:: bob.ip.binseg.data.imagefolder .. automodule:: bob.ip.binseg.data.imagefolderinference + Transforms ========== + .. note:: - All transforms work with :py:class:`PIL.Image.Image` objects. We make heavy use of the - `torchvision package`_ + + All transforms work with :py:class:`PIL.Image.Image` objects. We make heavy + use of the `torchvision package`_. .. automodule:: bob.ip.binseg.data.transforms + Losses ====== + .. automodule:: bob.ip.binseg.modeling.losses + Training ======== + .. automodule:: bob.ip.binseg.engine.trainer + Checkpointer ============ + .. automodule:: bob.ip.binseg.utils.checkpointer + Inference and Evaluation ======================== + .. automodule:: bob.ip.binseg.engine.inferencer + Plotting ======== + .. automodule:: bob.ip.binseg.utils.plot + .. include:: links.rst diff --git a/doc/configs.rst b/doc/configs.rst index e25e66c6afc3e2fc629c915f25475d101f81033c..93e09612f729f38396c3e69b63d6b02a50efe6f6 100644 --- a/doc/configs.rst +++ b/doc/configs.rst @@ -8,143 +8,169 @@ Configs Dataset Configs =============== +We provide variants for the training and test sets of each supported database, +as well as versions for COVD- (COmbined training sets of all publicly available +Vessel Dataset without target dataset) and SSL (Semi-supervised Learning), as +explained in our report. + .. _bob.ip.binseg.configs.datasets.imagefolder: ImageFolder ----------------- +----------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolder.py + .. _bob.ip.binseg.configs.datasets.imagefoldertest: ImageFolderTest ----------------- +--------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefoldertest.py + .. _bob.ip.binseg.configs.datasets.imagefolderinference: ImageFolderInference ---------------------- +-------------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolderinference.py + .. _bob.ip.binseg.configs.datasets.chasedb1: CHASEDB1 ----------------- +-------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1.py + +.. _bob.ip.binseg.configs.datasets.chasedb1test: + CHASEDB1TEST ----------------- +------------ .. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1test.py + .. _bob.ip.binseg.configs.datasets.covd-drive: COVD-DRIVE ----------------- +---------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py + .. _bob.ip.binseg.configs.datasets.covd-drive_ssl: COVD-DRIVE_SSL ----------------- +-------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py .. _bob.ip.binseg.configs.datasets.covd-stare: COVD-STARE ----------------- +---------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py + .. _bob.ip.binseg.configs.datasets.covd-stare_ssl: COVD-STARE_SSL ----------------- +-------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py .. _bob.ip.binseg.configs.datasets.covd-iostar: COVD-IOSTARVESSEL ----------------------- +----------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py + .. _bob.ip.binseg.configs.datasets.covd-iostar_ssl: COVD-IOSTARVESSEL_SSL ----------------------- +--------------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py .. _bob.ip.binseg.configs.datasets.covd-hrf: COVD-HRF ----------------------- +-------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py + .. _bob.ip.binseg.configs.datasets.covd-hrf_ssl: COVD-HRF_SSL ----------------------- +------------ .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py .. _bob.ip.binseg.configs.datasets.covd-chasedb1: COVD-CHASEDB1 ----------------------- +------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py + .. _bob.ip.binseg.configs.datasets.covd-chasedb1_ssl: COVD-CHASEDB1_SSL ----------------------- +----------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py .. _bob.ip.binseg.configs.datasets.drive: DRIVE ----------------------- +----- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drive.py +.. _bob.ip.binseg.configs.datasets.drivetest: + DRIVETEST ----------------------- +--------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/drivetest.py .. _bob.ip.binseg.configs.datasets.hrf: HRF ----------------------- +--- .. literalinclude:: ../bob/ip/binseg/configs/datasets/hrf1168.py + +.. _bob.ip.binseg.configs.datasets.hrftest: + HRFTEST ----------------------- +------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/hrftest.py - .. _bob.ip.binseg.configs.datasets.iostar: IOSTARVESSEL ----------------------- +------------ .. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvessel.py + +.. _bob.ip.binseg.configs.datasets.iostarvesseltest: + IOSTARVESSELTEST ----------------------- +---------------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvesseltest.py - .. _bob.ip.binseg.configs.datasets.stare: STARE ----------------------- +----- .. literalinclude:: ../bob/ip/binseg/configs/datasets/stare.py + +.. _bob.ip.binseg.configs.datasets.staretest: + STARETEST ----------------------- +--------- .. literalinclude:: ../bob/ip/binseg/configs/datasets/staretest.py @@ -152,10 +178,11 @@ STARETEST Model Configs ============== + .. _bob.ip.binseg.configs.models.driu: DRIU ------ +---- .. literalinclude:: ../bob/ip/binseg/configs/models/driu.py @@ -169,7 +196,7 @@ DRIUBN .. _bob.ip.binseg.configs.models.hed: HED ------ +--- .. literalinclude:: ../bob/ip/binseg/configs/models/hed.py @@ -183,16 +210,17 @@ M2UNet .. _bob.ip.binseg.configs.models.unet: UNet ------ +---- .. literalinclude:: ../bob/ip/binseg/configs/models/unet.py .. _bob.ip.binseg.configs.models.driussl: DRIUSSL --------- +------- .. literalinclude:: ../bob/ip/binseg/configs/models/driussl.py + .. _bob.ip.binseg.configs.models.driubnssl: DRIUBNSSL @@ -205,4 +233,3 @@ DRIUBNSSL M2UNetSSL --------- .. literalinclude:: ../bob/ip/binseg/configs/models/m2unetssl.py - diff --git a/doc/covdresults.rst b/doc/covdresults.rst index bf2305f41fc0d8dfbb6fc379af11702d51b5e6da..4eb4c1202a5a60f49553c87feb34a2ceaff33465 100644 --- a/doc/covdresults.rst +++ b/doc/covdresults.rst @@ -1,19 +1,20 @@ .. -*- coding: utf-8 -*- -.. _bob.ip.binseg.covdresults: +.. _bob.ip.binseg.covdresults: -========================== -COVD- and COVD-SLL Results -========================== +============================ + COVD- and COVD-SLL Results +============================ -In addition to the M2U-Net architecture, we also evaluated the larger DRIU network and a variation of it -that contains batch normalization (DRIU BN) on COVD- and COVD-SSL. Perhaps surprisingly, for the -majority of combinations, the performance of the DRIU variants are roughly equal or worse than the M2U-Net. -We anticipate that one reason for this could be overparameterization of large VGG16 models -that are pretrained on ImageNet. +In addition to the M2U-Net architecture, we also evaluated the larger DRIU +network and a variation of it that contains batch normalization (DRIU BN) on +COVD- and COVD-SSL. Perhaps surprisingly, for the majority of combinations, the +performance of the DRIU variants are roughly equal or worse than the M2U-Net. +We anticipate that one reason for this could be overparameterization of large +VGG16 models that are pretrained on ImageNet. F1 Scores -=========== +========= Comparison of F1-micro-scores (std) of DRIU and M2U-Net on COVD- and COVD-SSL. Standard deviation across test-images in brackets. @@ -44,8 +45,9 @@ Standard deviation across test-images in brackets. M2U-Net Precision vs. Recall Curves =================================== -Precision vs. recall curves for each evaluated dataset. -Note that here the F1-score is calculated on a macro level (see paper for more details). + +Precision vs. recall curves for each evaluated dataset. Note that here the +F1-score is calculated on a macro level (see paper for more details). .. figure:: img/pr_CHASEDB1.png :scale: 50 % diff --git a/doc/datasets.rst b/doc/datasets.rst index 5b82d3b12f8909a414055a17a3360c73d8449e1e..dd0f1cd3b715b5a74e33a42729c0a0fd2ededf9e 100644 --- a/doc/datasets.rst +++ b/doc/datasets.rst @@ -1,9 +1,10 @@ .. -*- coding: utf-8 -*- + .. _bob.ip.binseg.datasets: -================== -Supported Datasets -================== +==================== + Supported Datasets +==================== +-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+ | # | Name | H x W | # imgs | Train | Test | Mask | Vessel | OD | Cup | Train-Test split reference | @@ -33,32 +34,40 @@ Supported Datasets Add-on: Folder-based Dataset ============================ -For quick experimentation we also provide a PyTorch class that works with the following -dataset folder structure for images and ground-truth (gt): +For quick experimentation, we also provide a PyTorch_ class that works with the +following dataset folder structure for images and ground-truth (gt): -.. code-block:: bash +.. code-block:: text - root - |- images - |- gt + root + |- images + |- gt -the file names should have the same stem. Currently all image formats that can be read via PIL are supported. Additionally we support hdf5 binary files. -For training a new dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefolder` and amend accordingly, -e.g. the full path of the dataset and if necessary any preprocessing steps such as resizing, cropping, padding etc.. +The file names should have the same stem. Currently, all image formats that can +be read via PIL are supported. Additionally, we also support HDF5 binary +files. -Training can then be started with +For training, a new dataset configuration needs to be created. You can copy the +template :ref:`bob.ip.binseg.configs.datasets.imagefolder` and amend it +accordingly, e.g. to point to the the full path of the dataset and if necessary +any preprocessing steps such as resizing, cropping, padding etc. -.. code-block:: bash +Training can then be started with, e.g.: - bob binseg train M2UNet /path/to/myimagefolderconfig.py -b 4 -d cuda -o /my/output/path -vv +.. code-block:: sh -Similary for testing, a test dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefoldertest` and amend accordingly. + bob binseg train M2UNet /path/to/myimagefolderconfig.py -b 4 -d cuda -o /my/output/path -vv -Testing can then be started with +Similary for testing, a test dataset config needs to be created. You can copy +the template :ref:`bob.ip.binseg.configs.datasets.imagefoldertest` and amend it +accordingly. + +Testing can then be started with, e.g.: .. code-block:: bash - bob binseg test M2UNet /path/to/myimagefoldertestconfig.py -b 2 -d cuda -o /my/output/path -vv + bob binseg test M2UNet /path/to/myimagefoldertestconfig.py -b 2 -d cuda -o /my/output/path -vv + .. include:: links.rst diff --git a/doc/links.rst b/doc/links.rst index 12e8c5d3e7d6f2f144f888b8d2c3a2dde515e406..c11cdfe2b94706d60b1a94565d8086a77649e7fb 100644 --- a/doc/links.rst +++ b/doc/links.rst @@ -6,6 +6,7 @@ .. _bob: http://www.idiap.ch/software/bob .. _installation: https://www.idiap.ch/software/bob/docs/bob/docs/stable/bob/bob/doc/install.html .. _mailing list: https://www.idiap.ch/software/bob/discuss +.. _pytorch: https://pytorch.org .. _torchvision package: https://github.com/pytorch/vision .. DRIVE diff --git a/doc/setup.rst b/doc/setup.rst index 1200aac25c1074e9dc4695e0d78debdbcc995e5c..0f1a68d2b6b18e998caf5efafe84886e43720246 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -1,30 +1,38 @@ .. -*- coding: utf-8 -*- .. _bob.ip.binseg.setup: -========= -Setup -========= +======= + Setup +======= -Bob.ip.binseg -============= +Complete Bob's `installation`_ instructions. Then, to install this package, do +this: -Complete bob's `installation`_ instructions. Then, to install this -package +.. code-block:: sh + + $ conda activate <myenv> + (<myenv>) $ conda install bob.ip.binseg + +.. note:: -.. code-block:: bash + The value ``<myenv>`` should correspond to the name of the environment where + you initially installed your Bob packages. - conda install bob.ip.binseg Datasets -======== +-------- -The package supports a range of retina fundus datasets but does not install the `bob.db` -APIs by default, nor does it include the datasets. +The package supports a range of retina fundus datasets, but does not install the +`bob.db` iterator APIs by default, or includes the raw data itself, which you +must procure. -To setup a datasets: +To setup a dataset, do the following: -1. Download the dataset from the authors website -2. Install the corresponding bob.db package via ``conda install bob.db.<database>``. E.g. to install the DRIVE API run ``conda install bob.db.drive`` +1. Download the dataset from the authors website (see below for all download + links) +2. Install the corresponding bob.db package via ``conda install + bob.db.<database>``. E.g. to install the DRIVE API run ``conda install + bob.db.drive`` 3. :ref:`datasetpathsetup` 4. :ref:`dsconsistency` @@ -50,55 +58,59 @@ To setup a datasets: | REFUGE | https://refuge.grand-challenge.org/Details/ | `bob.db.refuge` | +------------+----------------------------------------------------------------------+---------------------+ + .. _datasetpathsetup: Set up dataset paths -===================== - -For each dataset that you are planning to use, set the datadir to -the path where it is stored. E.g.: +==================== -.. code-block:: bash +.. warning:: - bob config set bob.db.drive.datadir "/path/to/drivedataset/" + Our dataset connectors expect you provide "root" paths of raw datasets as + you unpack them in their **pristine** state. Changing the location of files + within a dataset distribution will likely cause execution errors. -To check your current setup +For each dataset that you are planning to use, set the ``datadir`` to the root +path where it is stored. E.g.: -.. code-block:: bash +.. code-block:: sh - bob config show + (<myenv>) $ bob config set bob.db.drive.datadir "/path/to/drivedataset/" -This should result in an output similar to the following: +To check your current setup, do the following: -.. code-block:: bash +.. code-block:: sh - { - "bob.db.chasedb1.datadir": "/idiap/resource/database/CHASE-DB11/", - "bob.db.drionsdb.datadir": "/idiap/resource/database/DRIONS", - "bob.db.drishtigs1.datadir": "/idiap/resource/database/Drishti-GS1/", - "bob.db.drive.datadir": "/idiap/resource/database/DRIVE", - "bob.db.hrf.datadir": "/idiap/resource/database/HRF", - "bob.db.iostar.datadir": "/idiap/resource/database/IOSTAR/IOSTAR Vessel Segmentation Dataset/", - "bob.db.refuge.datadir": "/idiap/resource/database/REFUGE", - "bob.db.rimoner3.datadir": "/idiap/resource/database/RIM-ONE/RIM-ONE r3", - "bob.db.stare.datadir": "/idiap/resource/database/STARE" - } + (<myenv>) $ bob config show + { + "bob.db.chasedb1.datadir": "/idiap/resource/database/CHASE-DB11/", + "bob.db.drionsdb.datadir": "/idiap/resource/database/DRIONS", + "bob.db.drishtigs1.datadir": "/idiap/resource/database/Drishti-GS1/", + "bob.db.drive.datadir": "/idiap/resource/database/DRIVE", + "bob.db.hrf.datadir": "/idiap/resource/database/HRF", + "bob.db.iostar.datadir": "/idiap/resource/database/IOSTAR/IOSTAR Vessel Segmentation Dataset/", + "bob.db.refuge.datadir": "/idiap/resource/database/REFUGE", + "bob.db.rimoner3.datadir": "/idiap/resource/database/RIM-ONE/RIM-ONE r3", + "bob.db.stare.datadir": "/idiap/resource/database/STARE" + } .. _dsconsistency: Test dataset consitency -======================== +======================= -To check whether the downloaded version is consistent with -the structure that is expected by our ``bob.db`` packages -run ``bob_dbmanage.py datasettocheck checkfiles`` -E.g.: +To check whether the downloaded version is consistent with the structure that +is expected by our ``bob.db`` packages, run ``bob_dbmanage.py datasettocheck +checkfiles`` E.g.: .. code-block:: sh - conda activate your-conda-env-with-bob.ip.binseg - bob_dbmanage.py drive checkfiles - > checkfiles completed sucessfully + (<myenv>) $ bob_dbmanage.py drive checkfiles + > checkfiles completed sucessfully + +If there are problems on the current file organisation, this procedure should +detect and highlight which files are missing. + .. include:: links.rst diff --git a/doc/training.rst b/doc/training.rst index 5e5c83a4ada67722c274039429bd4731d844c72c..4c8a1da1d1ba22c782081006f8046dc4a5b85453 100644 --- a/doc/training.rst +++ b/doc/training.rst @@ -6,13 +6,15 @@ Training ======== -To replicate our results use ``bob binseg train`` followed by the model config -and the dataset config. Use ``bob binseg train --help`` for more information. +To replicate our results use our main application ``bob binseg train`` followed +by the model configuration and dataset configuration files. Use ``bob binseg +train --help`` for more information. .. note:: - We strongly advice training with a GPU (using ``-d cuda``). Depending on the available GPU - memory you might have to adjust your batch size (``-b``). + We strongly advice training with a GPU (using ``-d cuda``). Depending on the + available GPU memory you might have to adjust your batch size (``-b``). + Default Dataset configs =======================