From 7329d79631b6a4bf6c7cb316002701e999d04aef Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 17 Mar 2020 20:46:26 +0100
Subject: [PATCH] [doc] Cleaning-up documentation
---
doc/api.rst | 21 ++++++++-
doc/configs.rst | 87 ++++++++++++++++++++++++-------------
doc/covdresults.rst | 26 +++++------
doc/datasets.rst | 45 +++++++++++--------
doc/links.rst | 1 +
doc/setup.rst | 102 +++++++++++++++++++++++++-------------------
doc/training.rst | 10 +++--
7 files changed, 181 insertions(+), 111 deletions(-)
diff --git a/doc/api.rst b/doc/api.rst
index 1eade482..2f732d29 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -11,40 +11,57 @@ run binary-segmentation benchmarks.
PyTorch bob.db Dataset
======================
+
.. automodule:: bob.ip.binseg.data.binsegdataset
+
PyTorch ImageFolder Dataset
===========================
+
.. automodule:: bob.ip.binseg.data.imagefolder
.. automodule:: bob.ip.binseg.data.imagefolderinference
+
Transforms
==========
+
.. note::
- All transforms work with :py:class:`PIL.Image.Image` objects. We make heavy use of the
- `torchvision package`_
+
+ All transforms work with :py:class:`PIL.Image.Image` objects. We make heavy
+ use of the `torchvision package`_.
.. automodule:: bob.ip.binseg.data.transforms
+
Losses
======
+
.. automodule:: bob.ip.binseg.modeling.losses
+
Training
========
+
.. automodule:: bob.ip.binseg.engine.trainer
+
Checkpointer
============
+
.. automodule:: bob.ip.binseg.utils.checkpointer
+
Inference and Evaluation
========================
+
.. automodule:: bob.ip.binseg.engine.inferencer
+
Plotting
========
+
.. automodule:: bob.ip.binseg.utils.plot
+
.. include:: links.rst
diff --git a/doc/configs.rst b/doc/configs.rst
index e25e66c6..93e09612 100644
--- a/doc/configs.rst
+++ b/doc/configs.rst
@@ -8,143 +8,169 @@ Configs
Dataset Configs
===============
+We provide variants for the training and test sets of each supported database,
+as well as versions for COVD- (COmbined training sets of all publicly available
+Vessel Dataset without target dataset) and SSL (Semi-supervised Learning), as
+explained in our report.
+
.. _bob.ip.binseg.configs.datasets.imagefolder:
ImageFolder
-----------------
+-----------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolder.py
+
.. _bob.ip.binseg.configs.datasets.imagefoldertest:
ImageFolderTest
-----------------
+---------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefoldertest.py
+
.. _bob.ip.binseg.configs.datasets.imagefolderinference:
ImageFolderInference
----------------------
+--------------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/imagefolderinference.py
+
.. _bob.ip.binseg.configs.datasets.chasedb1:
CHASEDB1
-----------------
+--------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1.py
+
+.. _bob.ip.binseg.configs.datasets.chasedb1test:
+
CHASEDB1TEST
-----------------
+------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/chasedb1test.py
+
.. _bob.ip.binseg.configs.datasets.covd-drive:
COVD-DRIVE
-----------------
+----------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544.py
+
.. _bob.ip.binseg.configs.datasets.covd-drive_ssl:
COVD-DRIVE_SSL
-----------------
+--------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/starechasedb1iostarhrf544ssldrive.py
.. _bob.ip.binseg.configs.datasets.covd-stare:
COVD-STARE
-----------------
+----------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608.py
+
.. _bob.ip.binseg.configs.datasets.covd-stare_ssl:
COVD-STARE_SSL
-----------------
+--------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivechasedb1iostarhrf608sslstare.py
.. _bob.ip.binseg.configs.datasets.covd-iostar:
COVD-IOSTARVESSEL
-----------------------
+-----------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024.py
+
.. _bob.ip.binseg.configs.datasets.covd-iostar_ssl:
COVD-IOSTARVESSEL_SSL
-----------------------
+---------------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1hrf1024ssliostar.py
.. _bob.ip.binseg.configs.datasets.covd-hrf:
COVD-HRF
-----------------------
+--------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168.py
+
.. _bob.ip.binseg.configs.datasets.covd-hrf_ssl:
COVD-HRF_SSL
-----------------------
+------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestarechasedb1iostar1168sslhrf.py
.. _bob.ip.binseg.configs.datasets.covd-chasedb1:
COVD-CHASEDB1
-----------------------
+-------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
+
.. _bob.ip.binseg.configs.datasets.covd-chasedb1_ssl:
COVD-CHASEDB1_SSL
-----------------------
+-----------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivestareiostarhrf960.py
.. _bob.ip.binseg.configs.datasets.drive:
DRIVE
-----------------------
+-----
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drive.py
+.. _bob.ip.binseg.configs.datasets.drivetest:
+
DRIVETEST
-----------------------
+---------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/drivetest.py
.. _bob.ip.binseg.configs.datasets.hrf:
HRF
-----------------------
+---
.. literalinclude:: ../bob/ip/binseg/configs/datasets/hrf1168.py
+
+.. _bob.ip.binseg.configs.datasets.hrftest:
+
HRFTEST
-----------------------
+-------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/hrftest.py
-
.. _bob.ip.binseg.configs.datasets.iostar:
IOSTARVESSEL
-----------------------
+------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvessel.py
+
+.. _bob.ip.binseg.configs.datasets.iostarvesseltest:
+
IOSTARVESSELTEST
-----------------------
+----------------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/iostarvesseltest.py
-
.. _bob.ip.binseg.configs.datasets.stare:
STARE
-----------------------
+-----
.. literalinclude:: ../bob/ip/binseg/configs/datasets/stare.py
+
+.. _bob.ip.binseg.configs.datasets.staretest:
+
STARETEST
-----------------------
+---------
.. literalinclude:: ../bob/ip/binseg/configs/datasets/staretest.py
@@ -152,10 +178,11 @@ STARETEST
Model Configs
==============
+
.. _bob.ip.binseg.configs.models.driu:
DRIU
------
+----
.. literalinclude:: ../bob/ip/binseg/configs/models/driu.py
@@ -169,7 +196,7 @@ DRIUBN
.. _bob.ip.binseg.configs.models.hed:
HED
------
+---
.. literalinclude:: ../bob/ip/binseg/configs/models/hed.py
@@ -183,16 +210,17 @@ M2UNet
.. _bob.ip.binseg.configs.models.unet:
UNet
------
+----
.. literalinclude:: ../bob/ip/binseg/configs/models/unet.py
.. _bob.ip.binseg.configs.models.driussl:
DRIUSSL
---------
+-------
.. literalinclude:: ../bob/ip/binseg/configs/models/driussl.py
+
.. _bob.ip.binseg.configs.models.driubnssl:
DRIUBNSSL
@@ -205,4 +233,3 @@ DRIUBNSSL
M2UNetSSL
---------
.. literalinclude:: ../bob/ip/binseg/configs/models/m2unetssl.py
-
diff --git a/doc/covdresults.rst b/doc/covdresults.rst
index bf2305f4..4eb4c120 100644
--- a/doc/covdresults.rst
+++ b/doc/covdresults.rst
@@ -1,19 +1,20 @@
.. -*- coding: utf-8 -*-
-.. _bob.ip.binseg.covdresults:
+.. _bob.ip.binseg.covdresults:
-==========================
-COVD- and COVD-SLL Results
-==========================
+============================
+ COVD- and COVD-SLL Results
+============================
-In addition to the M2U-Net architecture, we also evaluated the larger DRIU network and a variation of it
-that contains batch normalization (DRIU BN) on COVD- and COVD-SSL. Perhaps surprisingly, for the
-majority of combinations, the performance of the DRIU variants are roughly equal or worse than the M2U-Net.
-We anticipate that one reason for this could be overparameterization of large VGG16 models
-that are pretrained on ImageNet.
+In addition to the M2U-Net architecture, we also evaluated the larger DRIU
+network and a variation of it that contains batch normalization (DRIU BN) on
+COVD- and COVD-SSL. Perhaps surprisingly, for the majority of combinations, the
+performance of the DRIU variants are roughly equal or worse than the M2U-Net.
+We anticipate that one reason for this could be overparameterization of large
+VGG16 models that are pretrained on ImageNet.
F1 Scores
-===========
+=========
Comparison of F1-micro-scores (std) of DRIU and M2U-Net on COVD- and COVD-SSL.
Standard deviation across test-images in brackets.
@@ -44,8 +45,9 @@ Standard deviation across test-images in brackets.
M2U-Net Precision vs. Recall Curves
===================================
-Precision vs. recall curves for each evaluated dataset.
-Note that here the F1-score is calculated on a macro level (see paper for more details).
+
+Precision vs. recall curves for each evaluated dataset. Note that here the
+F1-score is calculated on a macro level (see paper for more details).
.. figure:: img/pr_CHASEDB1.png
:scale: 50 %
diff --git a/doc/datasets.rst b/doc/datasets.rst
index 5b82d3b1..dd0f1cd3 100644
--- a/doc/datasets.rst
+++ b/doc/datasets.rst
@@ -1,9 +1,10 @@
.. -*- coding: utf-8 -*-
+
.. _bob.ip.binseg.datasets:
-==================
-Supported Datasets
-==================
+====================
+ Supported Datasets
+====================
+-----+---------------+-------------+--------+-------+------+------+--------+-----+-----+----------------------------+
| # | Name | H x W | # imgs | Train | Test | Mask | Vessel | OD | Cup | Train-Test split reference |
@@ -33,32 +34,40 @@ Supported Datasets
Add-on: Folder-based Dataset
============================
-For quick experimentation we also provide a PyTorch class that works with the following
-dataset folder structure for images and ground-truth (gt):
+For quick experimentation, we also provide a PyTorch_ class that works with the
+following dataset folder structure for images and ground-truth (gt):
-.. code-block:: bash
+.. code-block:: text
- root
- |- images
- |- gt
+ root
+ |- images
+ |- gt
-the file names should have the same stem. Currently all image formats that can be read via PIL are supported. Additionally we support hdf5 binary files.
-For training a new dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefolder` and amend accordingly,
-e.g. the full path of the dataset and if necessary any preprocessing steps such as resizing, cropping, padding etc..
+The file names should have the same stem. Currently, all image formats that can
+be read via PIL are supported. Additionally, we also support HDF5 binary
+files.
-Training can then be started with
+For training, a new dataset configuration needs to be created. You can copy the
+template :ref:`bob.ip.binseg.configs.datasets.imagefolder` and amend it
+accordingly, e.g. to point to the the full path of the dataset and if necessary
+any preprocessing steps such as resizing, cropping, padding etc.
-.. code-block:: bash
+Training can then be started with, e.g.:
- bob binseg train M2UNet /path/to/myimagefolderconfig.py -b 4 -d cuda -o /my/output/path -vv
+.. code-block:: sh
-Similary for testing, a test dataset config needs to be created. You can copy the template :ref:`bob.ip.binseg.configs.datasets.imagefoldertest` and amend accordingly.
+ bob binseg train M2UNet /path/to/myimagefolderconfig.py -b 4 -d cuda -o /my/output/path -vv
-Testing can then be started with
+Similary for testing, a test dataset config needs to be created. You can copy
+the template :ref:`bob.ip.binseg.configs.datasets.imagefoldertest` and amend it
+accordingly.
+
+Testing can then be started with, e.g.:
.. code-block:: bash
- bob binseg test M2UNet /path/to/myimagefoldertestconfig.py -b 2 -d cuda -o /my/output/path -vv
+ bob binseg test M2UNet /path/to/myimagefoldertestconfig.py -b 2 -d cuda -o /my/output/path -vv
+
.. include:: links.rst
diff --git a/doc/links.rst b/doc/links.rst
index 12e8c5d3..c11cdfe2 100644
--- a/doc/links.rst
+++ b/doc/links.rst
@@ -6,6 +6,7 @@
.. _bob: http://www.idiap.ch/software/bob
.. _installation: https://www.idiap.ch/software/bob/docs/bob/docs/stable/bob/bob/doc/install.html
.. _mailing list: https://www.idiap.ch/software/bob/discuss
+.. _pytorch: https://pytorch.org
.. _torchvision package: https://github.com/pytorch/vision
.. DRIVE
diff --git a/doc/setup.rst b/doc/setup.rst
index 1200aac2..0f1a68d2 100644
--- a/doc/setup.rst
+++ b/doc/setup.rst
@@ -1,30 +1,38 @@
.. -*- coding: utf-8 -*-
.. _bob.ip.binseg.setup:
-=========
-Setup
-=========
+=======
+ Setup
+=======
-Bob.ip.binseg
-=============
+Complete Bob's `installation`_ instructions. Then, to install this package, do
+this:
-Complete bob's `installation`_ instructions. Then, to install this
-package
+.. code-block:: sh
+
+ $ conda activate <myenv>
+ (<myenv>) $ conda install bob.ip.binseg
+
+.. note::
-.. code-block:: bash
+ The value ``<myenv>`` should correspond to the name of the environment where
+ you initially installed your Bob packages.
- conda install bob.ip.binseg
Datasets
-========
+--------
-The package supports a range of retina fundus datasets but does not install the `bob.db`
-APIs by default, nor does it include the datasets.
+The package supports a range of retina fundus datasets, but does not install the
+`bob.db` iterator APIs by default, or includes the raw data itself, which you
+must procure.
-To setup a datasets:
+To setup a dataset, do the following:
-1. Download the dataset from the authors website
-2. Install the corresponding bob.db package via ``conda install bob.db.<database>``. E.g. to install the DRIVE API run ``conda install bob.db.drive``
+1. Download the dataset from the authors website (see below for all download
+ links)
+2. Install the corresponding bob.db package via ``conda install
+ bob.db.<database>``. E.g. to install the DRIVE API run ``conda install
+ bob.db.drive``
3. :ref:`datasetpathsetup`
4. :ref:`dsconsistency`
@@ -50,55 +58,59 @@ To setup a datasets:
| REFUGE | https://refuge.grand-challenge.org/Details/ | `bob.db.refuge` |
+------------+----------------------------------------------------------------------+---------------------+
+
.. _datasetpathsetup:
Set up dataset paths
-=====================
-
-For each dataset that you are planning to use, set the datadir to
-the path where it is stored. E.g.:
+====================
-.. code-block:: bash
+.. warning::
- bob config set bob.db.drive.datadir "/path/to/drivedataset/"
+ Our dataset connectors expect you provide "root" paths of raw datasets as
+ you unpack them in their **pristine** state. Changing the location of files
+ within a dataset distribution will likely cause execution errors.
-To check your current setup
+For each dataset that you are planning to use, set the ``datadir`` to the root
+path where it is stored. E.g.:
-.. code-block:: bash
+.. code-block:: sh
- bob config show
+ (<myenv>) $ bob config set bob.db.drive.datadir "/path/to/drivedataset/"
-This should result in an output similar to the following:
+To check your current setup, do the following:
-.. code-block:: bash
+.. code-block:: sh
- {
- "bob.db.chasedb1.datadir": "/idiap/resource/database/CHASE-DB11/",
- "bob.db.drionsdb.datadir": "/idiap/resource/database/DRIONS",
- "bob.db.drishtigs1.datadir": "/idiap/resource/database/Drishti-GS1/",
- "bob.db.drive.datadir": "/idiap/resource/database/DRIVE",
- "bob.db.hrf.datadir": "/idiap/resource/database/HRF",
- "bob.db.iostar.datadir": "/idiap/resource/database/IOSTAR/IOSTAR Vessel Segmentation Dataset/",
- "bob.db.refuge.datadir": "/idiap/resource/database/REFUGE",
- "bob.db.rimoner3.datadir": "/idiap/resource/database/RIM-ONE/RIM-ONE r3",
- "bob.db.stare.datadir": "/idiap/resource/database/STARE"
- }
+ (<myenv>) $ bob config show
+ {
+ "bob.db.chasedb1.datadir": "/idiap/resource/database/CHASE-DB11/",
+ "bob.db.drionsdb.datadir": "/idiap/resource/database/DRIONS",
+ "bob.db.drishtigs1.datadir": "/idiap/resource/database/Drishti-GS1/",
+ "bob.db.drive.datadir": "/idiap/resource/database/DRIVE",
+ "bob.db.hrf.datadir": "/idiap/resource/database/HRF",
+ "bob.db.iostar.datadir": "/idiap/resource/database/IOSTAR/IOSTAR Vessel Segmentation Dataset/",
+ "bob.db.refuge.datadir": "/idiap/resource/database/REFUGE",
+ "bob.db.rimoner3.datadir": "/idiap/resource/database/RIM-ONE/RIM-ONE r3",
+ "bob.db.stare.datadir": "/idiap/resource/database/STARE"
+ }
.. _dsconsistency:
Test dataset consitency
-========================
+=======================
-To check whether the downloaded version is consistent with
-the structure that is expected by our ``bob.db`` packages
-run ``bob_dbmanage.py datasettocheck checkfiles``
-E.g.:
+To check whether the downloaded version is consistent with the structure that
+is expected by our ``bob.db`` packages, run ``bob_dbmanage.py datasettocheck
+checkfiles`` E.g.:
.. code-block:: sh
- conda activate your-conda-env-with-bob.ip.binseg
- bob_dbmanage.py drive checkfiles
- > checkfiles completed sucessfully
+ (<myenv>) $ bob_dbmanage.py drive checkfiles
+ > checkfiles completed sucessfully
+
+If there are problems on the current file organisation, this procedure should
+detect and highlight which files are missing.
+
.. include:: links.rst
diff --git a/doc/training.rst b/doc/training.rst
index 5e5c83a4..4c8a1da1 100644
--- a/doc/training.rst
+++ b/doc/training.rst
@@ -6,13 +6,15 @@
Training
========
-To replicate our results use ``bob binseg train`` followed by the model config
-and the dataset config. Use ``bob binseg train --help`` for more information.
+To replicate our results use our main application ``bob binseg train`` followed
+by the model configuration and dataset configuration files. Use ``bob binseg
+train --help`` for more information.
.. note::
- We strongly advice training with a GPU (using ``-d cuda``). Depending on the available GPU
- memory you might have to adjust your batch size (``-b``).
+ We strongly advice training with a GPU (using ``-d cuda``). Depending on the
+ available GPU memory you might have to adjust your batch size (``-b``).
+
Default Dataset configs
=======================
--
GitLab