From ba9be5b4d3a68009165855a97e2dcb73e07a67a5 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Thu, 2 Apr 2020 20:48:07 +0200
Subject: [PATCH] [doc] Documentation improvements

---
 bob/ip/binseg/script/binseg.py | 242 -------------------
 bob/ip/binseg/script/config.py |   2 -
 bob/ip/binseg/script/train.py  | 240 +++++++++++++++++++
 conda/meta.yaml                |   8 +-
 doc/api.rst                    |  26 +-
 doc/cli.rst                    |  74 ++++++
 doc/conf.py                    |   6 +
 doc/index.rst                  |   5 +-
 doc/training.rst               | 424 ++++++++++-----------------------
 doc/usage.rst                  |  34 ++-
 setup.py                       |   3 +-
 11 files changed, 479 insertions(+), 585 deletions(-)
 create mode 100644 bob/ip/binseg/script/train.py
 create mode 100644 doc/cli.rst

diff --git a/bob/ip/binseg/script/binseg.py b/bob/ip/binseg/script/binseg.py
index 88a995d1..3db339a7 100644
--- a/bob/ip/binseg/script/binseg.py
+++ b/bob/ip/binseg/script/binseg.py
@@ -22,8 +22,6 @@ from bob.extension.scripts.click_helper import (
 
 from bob.ip.binseg.utils.checkpointer import DetectronCheckpointer
 from torch.utils.data import DataLoader
-from bob.ip.binseg.engine.trainer import do_train
-from bob.ip.binseg.engine.ssltrainer import do_ssltrain
 from bob.ip.binseg.engine.inferencer import do_inference
 from bob.ip.binseg.utils.plot import plot_overview
 from bob.ip.binseg.utils.click import OptionEatAll
@@ -42,130 +40,6 @@ def binseg():
     """Binary 2D Image Segmentation Benchmark commands."""
 
 
-# Train
-@binseg.command(entry_point_group="bob.ip.binseg.config", cls=ConfigCommand,
-    epilog="""
-Examples:
-
-  1. Builds recipe from one of our build dependencies (inside bob.conda):
-
-\b
-     $ cd bob.conda
-     $ bdt build -vv conda/libblitz
-
-
-  2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
-
-     $ bdt build --python=3.6 -vv path/to/conda/dir
-
-
-  3. To build multiple recipes, just pass the paths to them:
-
-     $ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
-"""
-        )
-@click.option(
-    "--output-path", "-o", required=True, default="output", cls=ResourceOption
-)
-@click.option("--model", "-m", required=True, cls=ResourceOption)
-@click.option("--dataset", "-d", required=True, cls=ResourceOption)
-@click.option("--optimizer", required=True, cls=ResourceOption)
-@click.option("--criterion", required=True, cls=ResourceOption)
-@click.option("--scheduler", required=True, cls=ResourceOption)
-@click.option("--pretrained-backbone", "-t", required=True, cls=ResourceOption)
-@click.option("--batch-size", "-b", required=True, default=2, cls=ResourceOption)
-@click.option(
-    "--epochs",
-    "-e",
-    help="Number of epochs used for training",
-    show_default=True,
-    required=True,
-    default=1000,
-    cls=ResourceOption,
-)
-@click.option(
-    "--checkpoint-period",
-    "-p",
-    help="Number of epochs after which a checkpoint is saved",
-    show_default=True,
-    required=True,
-    default=100,
-    cls=ResourceOption,
-)
-@click.option(
-    "--device",
-    "-d",
-    help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
-    show_default=True,
-    required=True,
-    default="cpu",
-    cls=ResourceOption,
-)
-@click.option(
-    "--seed",
-    "-s",
-    help="torch random seed",
-    show_default=True,
-    required=False,
-    default=42,
-    cls=ResourceOption,
-)
-@verbosity_option(cls=ResourceOption)
-def train(
-    model,
-    optimizer,
-    scheduler,
-    output_path,
-    epochs,
-    pretrained_backbone,
-    batch_size,
-    criterion,
-    dataset,
-    checkpoint_period,
-    device,
-    seed,
-    **kwargs
-):
-    """ Train a model """
-
-    if not os.path.exists(output_path):
-        os.makedirs(output_path)
-    torch.manual_seed(seed)
-    # PyTorch dataloader
-    data_loader = DataLoader(
-        dataset=dataset,
-        batch_size=batch_size,
-        shuffle=True,
-        pin_memory=torch.cuda.is_available(),
-    )
-
-    # Checkpointer
-    checkpointer = DetectronCheckpointer(
-        model, optimizer, scheduler, save_dir=output_path, save_to_disk=True
-    )
-    arguments = {}
-    arguments["epoch"] = 0
-    extra_checkpoint_data = checkpointer.load(pretrained_backbone)
-    arguments.update(extra_checkpoint_data)
-    arguments["max_epoch"] = epochs
-
-    # Train
-    logger.info("Training for {} epochs".format(arguments["max_epoch"]))
-    logger.info("Continuing from epoch {}".format(arguments["epoch"]))
-    do_train(
-        model,
-        data_loader,
-        optimizer,
-        criterion,
-        scheduler,
-        checkpointer,
-        checkpoint_period,
-        device,
-        arguments,
-        output_path,
-    )
-
-
 # Inference
 @binseg.command(entry_point_group="bob.ip.binseg.config", cls=ConfigCommand)
 @click.option(
@@ -282,122 +156,6 @@ def visualize(dataset, output_path, **kwargs):
     logger.info("Saving transformed test images {}".format(output_path))
     savetransformedtest(dataset=dataset, output_path=output_path)
 
-
-# SSLTrain
-@binseg.command(entry_point_group="bob.ip.binseg.config", cls=ConfigCommand)
-@click.option(
-    "--output-path", "-o", required=True, default="output", cls=ResourceOption
-)
-@click.option("--model", "-m", required=True, cls=ResourceOption)
-@click.option("--dataset", "-d", required=True, cls=ResourceOption)
-@click.option("--optimizer", required=True, cls=ResourceOption)
-@click.option("--criterion", required=True, cls=ResourceOption)
-@click.option("--scheduler", required=True, cls=ResourceOption)
-@click.option("--pretrained-backbone", "-t", required=True, cls=ResourceOption)
-@click.option("--batch-size", "-b", required=True, default=2, cls=ResourceOption)
-@click.option(
-    "--epochs",
-    "-e",
-    help="Number of epochs used for training",
-    show_default=True,
-    required=True,
-    default=1000,
-    cls=ResourceOption,
-)
-@click.option(
-    "--checkpoint-period",
-    "-p",
-    help="Number of epochs after which a checkpoint is saved",
-    show_default=True,
-    required=True,
-    default=100,
-    cls=ResourceOption,
-)
-@click.option(
-    "--device",
-    "-d",
-    help='A string indicating the device to use (e.g. "cpu" or "cuda:0"',
-    show_default=True,
-    required=True,
-    default="cpu",
-    cls=ResourceOption,
-)
-@click.option(
-    "--rampup",
-    "-r",
-    help="Ramp-up length in epochs",
-    show_default=True,
-    required=True,
-    default="900",
-    cls=ResourceOption,
-)
-@click.option(
-    "--seed",
-    "-s",
-    help="torch random seed",
-    show_default=True,
-    required=False,
-    default=42,
-    cls=ResourceOption,
-)
-@verbosity_option(cls=ResourceOption)
-def ssltrain(
-    model,
-    optimizer,
-    scheduler,
-    output_path,
-    epochs,
-    pretrained_backbone,
-    batch_size,
-    criterion,
-    dataset,
-    checkpoint_period,
-    device,
-    rampup,
-    seed,
-    **kwargs
-):
-    """ Train a model """
-
-    if not os.path.exists(output_path):
-        os.makedirs(output_path)
-    torch.manual_seed(seed)
-    # PyTorch dataloader
-    data_loader = DataLoader(
-        dataset=dataset,
-        batch_size=batch_size,
-        shuffle=True,
-        pin_memory=torch.cuda.is_available(),
-    )
-
-    # Checkpointer
-    checkpointer = DetectronCheckpointer(
-        model, optimizer, scheduler, save_dir=output_path, save_to_disk=True
-    )
-    arguments = {}
-    arguments["epoch"] = 0
-    extra_checkpoint_data = checkpointer.load(pretrained_backbone)
-    arguments.update(extra_checkpoint_data)
-    arguments["max_epoch"] = epochs
-
-    # Train
-    logger.info("Training for {} epochs".format(arguments["max_epoch"]))
-    logger.info("Continuing from epoch {}".format(arguments["epoch"]))
-    do_ssltrain(
-        model,
-        data_loader,
-        optimizer,
-        criterion,
-        scheduler,
-        checkpointer,
-        checkpoint_period,
-        device,
-        arguments,
-        output_path,
-        rampup,
-    )
-
-
 # Apply image transforms to a folder containing images
 @binseg.command(entry_point_group="bob.ip.binseg.config", cls=ConfigCommand)
 @click.option("--source-path", "-s", required=True, cls=ResourceOption)
diff --git a/bob/ip/binseg/script/config.py b/bob/ip/binseg/script/config.py
index 72ceb2b4..e00f2fa5 100644
--- a/bob/ip/binseg/script/config.py
+++ b/bob/ip/binseg/script/config.py
@@ -6,7 +6,6 @@ import inspect
 
 import click
 import pkg_resources
-from click_plugins import with_plugins
 
 from bob.extension.scripts.click_helper import (
     verbosity_option,
@@ -14,7 +13,6 @@ from bob.extension.scripts.click_helper import (
 )
 
 import logging
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/bob/ip/binseg/script/train.py b/bob/ip/binseg/script/train.py
new file mode 100644
index 00000000..4e7c9523
--- /dev/null
+++ b/bob/ip/binseg/script/train.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import pkg_resources
+
+import click
+from click_plugins import with_plugins
+
+import torch
+from torch.utils.data import DataLoader
+
+from bob.extension.scripts.click_helper import (
+    verbosity_option,
+    ConfigCommand,
+    ResourceOption,
+    AliasedGroup,
+)
+
+from ..utils.checkpointer import DetectronCheckpointer
+from ..engine.trainer import do_train
+from ..engine.ssltrainer import do_ssltrain
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@click.command(
+    entry_point_group="bob.ip.binseg.config",
+    cls=ConfigCommand,
+    epilog="""Examples:
+
+\b
+    1. Trains a U-Net model (VGG-16 backbone) with DRIVE (vessel segmentation),
+       on a GPU (``cuda:0``):
+
+       $ bob binseg train -vv unet drive --batch-size=4 --device="cuda:0"
+
+    2. Trains a HED model with HRF on a GPU (``cuda:0``):
+
+       $ bob binseg train -vv hed hrf --batch-size=8 --device="cuda:0"
+
+    3. Trains a M2U-Net model on the COVD-DRIVE dataset on the CPU:
+
+       $ bob binseg train -vv m2unet covd-drive --batch-size=8
+
+    4. Trains a DRIU model with SSL on the COVD-HRF dataset on the CPU:
+
+       $ bob binseg train -vv --ssl driu-ssl covd-drive-ssl --batch-size=1
+
+""",
+)
+@click.option(
+    "--output-path",
+    "-o",
+    help="Path where to store the generated model (created if does not exist)",
+    required=True,
+    default="results",
+    cls=ResourceOption,
+)
+@click.option(
+    "--model",
+    "-m",
+    help="A torch.nn.Module instance implementing the network to be trained",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--dataset",
+    "-d",
+    help="A torch.utils.data.dataset.Dataset instance implementing a dataset to be used for training the model, possibly including all pre-processing pipelines required.",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--optimizer",
+    help="A torch.optim.Optimizer that will be used to train the network",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--criterion",
+    help="A loss function to compute the FCN error for every sample respecting the PyTorch API for loss functions (see torch.nn.modules.loss)",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--scheduler",
+    help="A learning rate scheduler that drives changes in the learning rate depending on the FCN state (see torch.optim.lr_scheduler)",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--pretrained-backbone",
+    "-t",
+    help="URLs of a pre-trained model file that will be used to preset FCN weights (where relevant) before training starts.  (e.g. vgg-16)",
+    required=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--batch-size",
+    "-b",
+    help="Number of samples in every batch (notice that changing this parameter affects memory requirements for the network)",
+    required=True,
+    show_default=True,
+    default=2,
+    cls=ResourceOption,
+)
+@click.option(
+    "--epochs",
+    "-e",
+    help="Number of epochs used for training",
+    show_default=True,
+    required=True,
+    default=1000,
+    cls=ResourceOption,
+)
+@click.option(
+    "--checkpoint-period",
+    "-p",
+    help="Number of epochs after which a checkpoint is saved",
+    show_default=True,
+    required=True,
+    default=100,
+    cls=ResourceOption,
+)
+@click.option(
+    "--device",
+    "-d",
+    help='A string indicating the device to use (e.g. "cpu" or "cuda:0")',
+    show_default=True,
+    required=True,
+    default="cpu",
+    cls=ResourceOption,
+)
+@click.option(
+    "--seed",
+    "-s",
+    help="Seed to use for the random number generator",
+    show_default=True,
+    required=False,
+    default=42,
+    cls=ResourceOption,
+)
+@click.option(
+    "--ssl/--no-ssl",
+    help="Switch ON/OFF semi-supervised training mode",
+    show_default=True,
+    required=True,
+    default=False,
+    cls=ResourceOption,
+)
+@click.option(
+    "--rampup",
+    "-r",
+    help="Ramp-up length in epochs (for SSL training only)",
+    show_default=True,
+    required=True,
+    default=900,
+    cls=ResourceOption,
+)
+@verbosity_option(cls=ResourceOption)
+def train(
+    model,
+    optimizer,
+    scheduler,
+    output_path,
+    epochs,
+    pretrained_backbone,
+    batch_size,
+    criterion,
+    dataset,
+    checkpoint_period,
+    device,
+    seed,
+    ssl,
+    rampup,
+    verbose,
+):
+    """Trains an FCN to perform binary segmentation using a supervised approach
+
+    Training is performed for a fixed number of steps (not configurable).
+    """
+
+    if not os.path.exists(output_path):
+        os.makedirs(output_path)
+    torch.manual_seed(seed)
+
+    # PyTorch dataloader
+    data_loader = DataLoader(
+        dataset=dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # Checkpointer
+    checkpointer = DetectronCheckpointer(
+        model, optimizer, scheduler, save_dir=output_path, save_to_disk=True
+    )
+    arguments = {}
+    arguments["epoch"] = 0
+    extra_checkpoint_data = checkpointer.load(pretrained_backbone)
+    arguments.update(extra_checkpoint_data)
+    arguments["max_epoch"] = epochs
+
+    logger.info("Training for {} epochs".format(arguments["max_epoch"]))
+    logger.info("Continuing from epoch {}".format(arguments["epoch"]))
+
+    if not ssl:
+        logger.info("Doing SUPERVISED training...")
+        do_train(
+            model,
+            data_loader,
+            optimizer,
+            criterion,
+            scheduler,
+            checkpointer,
+            checkpoint_period,
+            device,
+            arguments,
+            output_path,
+        )
+
+    else:
+
+        logger.info("Doing SEMI-SUPERVISED training...")
+        do_ssltrain(
+            model,
+            data_loader,
+            optimizer,
+            criterion,
+            scheduler,
+            checkpointer,
+            checkpoint_period,
+            device,
+            arguments,
+            output_path,
+            rampup,
+        )
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 2a29081e..9dff4279 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -56,13 +56,12 @@ test:
     - bob binseg config describe drive -v
     - bob binseg config copy --help
     - bob binseg config copy drive /tmp/test.py
+    - bob binseg train --help
+    - bob binseg test --help
+    - bob binseg predict --help
     - bob binseg compare --help
     - bob binseg evalpred --help
     - bob binseg gridtable --help
-    - bob binseg predict --help
-    - bob binseg ssltrain --help
-    - bob binseg test --help
-    - bob binseg train --help
     - bob binseg transformfolder --help
     - bob binseg visualize --help
     - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
@@ -76,6 +75,7 @@ test:
     - coverage
     - sphinx
     - sphinx_rtd_theme
+    - sphinxcontrib-programoutput
     - bob.db.drive
     - bob.db.stare
     - bob.db.chasedb1
diff --git a/doc/api.rst b/doc/api.rst
index f9dbddb8..b0e5fb83 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,18 +1,13 @@
 .. -*- coding: utf-8 -*-
 
-=====
- API
-=====
+=====================================
+ Application Program Interface (API)
+=====================================
 
 .. To update these lists, run the following command on the root of the package:
 .. find bob -name '*.py' | sed -e 's#/#.#g;s#.py$##g;s#.__init__##g' | sort
 .. You may apply further filtering to update only one of the subsections below
 
-.. autosummary::
-   :toctree: api/base
-
-   bob.ip.binseg
-
 
 Data Manipulation
 -----------------
@@ -83,21 +78,16 @@ Toolbox
    bob.ip.binseg.utils.transformfolder
 
 
-Scripts
--------
-
-.. autosummary::
-   :toctree: api/scripts
-
-   bob.ip.binseg.script
-   bob.ip.binseg.script.binseg
-
-
 .. _bob.ip.binseg.configs:
 
 Preset Configurations
 ---------------------
 
+Preset configurations for baseline systems
+
+This module contains preset configurations for baseline FCN architectures and
+datasets.
+
 
 Models
 ======
diff --git a/doc/cli.rst b/doc/cli.rst
new file mode 100644
index 00000000..47bdfc80
--- /dev/null
+++ b/doc/cli.rst
@@ -0,0 +1,74 @@
+.. -*- coding: utf-8 -*-
+
+.. _bob.ip.binseg.cli:
+
+==============================
+ Command-Line Interface (CLI)
+==============================
+
+This package provides a single entry point for all of its applications using
+:ref:`Bob's unified CLI mechanism <bob.extension.cli>`.  A list of available
+applications can be retrieved using:
+
+.. command-output:: bob binseg --help
+
+
+Preset Configuration Resources
+------------------------------
+
+A CLI application allows one to list, inspect and copy available configuration
+resources exported by this package.
+
+.. _bob.ip.binseg.cli.config:
+
+.. command-output:: bob binseg config --help
+
+
+.. _bob.ip.binseg.cli.config.list:
+
+Listing Resources
+=================
+
+.. command-output:: bob binseg config list --help
+
+
+.. _bob.ip.binseg.cli.config.list.all:
+
+Available Resources
+===================
+
+Here is a list of all resources currently exported.
+
+.. command-output:: bob binseg config list -v
+
+
+.. _bob.ip.binseg.cli.config.describe:
+
+Describing a Resource
+=====================
+
+.. command-output:: bob binseg config describe --help
+
+
+.. _bob.ip.binseg.cli.config.copy:
+
+Copying a Resource
+==================
+
+You may use this command to locally copy a resource file so you can change it.
+
+.. command-output:: bob binseg config copy --help
+
+
+.. _bob.ip.binseg.cli.train:
+
+Training FCNs
+-------------
+
+Training creates of a new PyTorch_ model.  This model can be used for
+evaluation tests or for inference.
+
+.. command-output:: bob binseg train --help
+
+
+.. include:: links.rst
diff --git a/doc/conf.py b/doc/conf.py
index 8abecdd1..2017d7b2 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -23,9 +23,15 @@ extensions = [
     'sphinx.ext.napoleon',
     'sphinx.ext.viewcode',
     'sphinx.ext.mathjax',
+    'sphinxcontrib.programoutput',
     #'matplotlib.sphinxext.plot_directive'
 ]
 
+# This allows sphinxcontrib-programoutput to work in buildout mode
+candidate_binpath = os.path.join(os.path.dirname(os.path.realpath(os.curdir)), 'bin')
+if os.path.exists(candidate_binpath):
+    os.environ['PATH'] = candidate_binpath + os.pathsep + os.environ.get('PATH', '')
+
 # Be picky about warnings
 nitpicky = True
 
diff --git a/doc/index.rst b/doc/index.rst
index e8fe3faa..2e11ada6 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -35,8 +35,8 @@ The additional material referred to in the paper can be found under
 .. todolist::
 
 
-Users Guide
------------
+User Guide
+----------
 
 .. toctree::
    :maxdepth: 2
@@ -48,6 +48,7 @@ Users Guide
    acknowledgements
    references
    datasets
+   cli
    api
 
 
diff --git a/doc/training.rst b/doc/training.rst
index 30e483f4..8b0eaef4 100644
--- a/doc/training.rst
+++ b/doc/training.rst
@@ -18,316 +18,132 @@ command-line options.  Use ``bob binseg train --help`` for more information.
 
 .. note::
 
-   We strongly advice training with a GPU (using ``-d cuda``). Depending on the
-   available GPU memory you might have to adjust your batch size (``-b``).
-
-
-Default Dataset configs
-=======================
-
-1. Vessel:
-
-* CHASEDB1
-* CHASEDB1TEST
-* COVD-DRIVE
-* COVD-DRIVE_SSL
-* COVD-STARE
-* COVD-STARE_SSL
-* COVD-IOSTARVESSEL
-* COVD-IOSTARVESSEL_SSL
-* COVD-HRF
-* COVD-HRF_SSL
-* COVD-CHASEDB1
-* COVD-CHASEDB1_SSL
-* DRIVE
-* DRIVETEST
-* HRF
-* HRFTEST
-* IOSTARVESSEL
-* IOSTARVESSELTEST
-* STARE
-* STARETEST
-
-2. Optic Disc and Cup
-
-* DRIONSDB
-* DRIONSDBTEST
-* DRISHTIGS1OD
-* DRISHTIGS1ODTEST
-* DRISHTIGS1CUP
-* DRISHTIGS1CUPTEST
-* IOSTAROD
-* IOSTARODTEST
-* REFUGECUP
-* REFUGECUPTEST
-* REFUGEOD
-* REFUGEODTEST
-* RIMONER3CUP
-* RIMONER3CUPTEST
-* RIMONER3OD
-* RIMONER3ODTEST
-
-Default Model configs
-=====================
-
-* DRIU
-* DRIUBN
-* DRIUSSL
-* DRIUBNSSL
-* DRIUOD
-* HED
-* M2UNet
-* M2UNetSSL
-* UNet
-
+   We strongly advice training with a GPU (using ``--device="cuda:0"``).
+   Depending on the available GPU memory you might have to adjust your batch
+   size (``--batch``).
 
 Baseline Benchmarks
 ===================
 
-.. code-block:: bash
-
-    #!/bin/bash
-    # set output directory
-    outputroot=`pwd`"/output"
-    mkdir -p $outputroot
-
-    #### Global config ####
-    m2u=M2UNet
-    hed=HED
-    driu=DRIU
-    unet=UNet
-    m2ussl=M2UNetSSL
-    driussl=DRIUSSL
-
-    #### CHASE_DB 1 ####
-    dataset=CHASEDB1
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # batch sizes
-    b_m2u=6
-    b_hed=4
-    b_driu=4
-    b_unet=2
-    # Train
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-    bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
-    #### DRIVE ####
-    dataset=DRIVE
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    b_m2u=16
-    b_hed=8
-    b_driu=8
-    b_unet=4
-    # Train
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-    bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
-    #### HRF ####
-    dataset=HRF
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    b_m2u=1
-    b_hed=1
-    b_driu=1
-    b_unet=1
-    # Train
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-    bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
-    #### IOSTAR VESSEL ####
-    dataset=IOSTARVESSEL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    b_m2u=6
-    b_hed=4
-    b_driu=4
-    b_unet=2
-    # Train
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-    bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
-
-    #### STARE ####
-    dataset=STARE
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    b_m2u=6
-    b_hed=4
-    b_driu=5
-    b_unet=2
-    # Train
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-    bob binseg train $hed $dataset -b $b_hed -d cuda -o $output"/"$hed -vv
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $unet $dataset -b $b_unet -d cuda -o $output"/"$unet -vv
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for supervised training of baselines.  Use it like this:
+
+.. code-block:: sh
+
+   # change <model> and <dataset> by one of items bellow
+   $ bob binseg train -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+  * - **Models / Datasets**
+    - :py:mod:`drive <bob.ip.binseg.configs.datasets.drive>`
+    - :py:mod:`stare <bob.ip.binseg.configs.datasets.stare>`
+    - :py:mod:`chasedb1 <bob.ip.binseg.configs.datasets.chasedb1>`
+    - :py:mod:`iostar-vessel <bob.ip.binseg.configs.datasets.iostarvessel>`
+    - :py:mod:`hrf <bob.ip.binseg.configs.datasets.hrf1168>`
+  * - :py:mod:`unet <bob.ip.binseg.configs.models.unet>`
+    - 4
+    - 2
+    - 2
+    - 2
+    - 1
+  * - :py:mod:`hed <bob.ip.binseg.configs.models.hed>`
+    - 8
+    - 4
+    - 4
+    - 4
+    - 1
+  * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driubn>`
+    - 8
+    - 5
+    - 4
+    - 4
+    - 1
+  * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+    - 16
+    - 6
+    - 6
+    - 6
+    - 1
+
+
+.. tip::
+
+   Instead of the default configurations, you can pass the full path of your
+   customized dataset and model files.  You may :ref:`copy any of the existing
+   configuration resources <bob.ip.binseg.cli.config.copy>` and change them
+   locally.  Once you're happy, you may use the newly created files directly on
+   your training command line.  For example, suppose you wanted to slightly
+   change the drive pre-processing pipeline.  You could do the following:
+
+   .. code-block:: bash
+
+      $ bob binseg config copy drive my_drive_remix.py
+      # edit my_drive_remix.py to your needs
+      $ bob binseg train -vv <model> ./my_drive_remix.py --batch-size=<see-table> --device="cuda:0"
+
+
+Combined Vessel Dataset (COVD)
+==============================
+
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for supervised training of COVD- systems.  Use it like this:
+
+.. code-block:: sh
+
+   # change <model> and <dataset> by one of items bellow
+   $ bob binseg train -vv <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+  * - **Models / Datasets**
+    - :py:mod:`covd-drive <bob.ip.binseg.configs.datasets.starechasedb1iostarhrf544>`
+    - :py:mod:`covd-stare <bob.ip.binseg.configs.datasets.drivechasedb1iostarhrf608>`
+    - :py:mod:`covd-chasedb1 <bob.ip.binseg.configs.datasets.drivestareiostarhrf960>`
+    - :py:mod:`covd-iostar-vessel <bob.ip.binseg.configs.datasets.drivestarechasedb1hrf1024>`
+    - :py:mod:`covd-hrf <bob.ip.binseg.configs.datasets.drivestarechasedb1iostar1168>`
+  * - :py:mod:`driu <bob.ip.binseg.configs.models.driu>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driubn>`
+    - 4
+    - 4
+    - 2
+    - 2
+    - 2
+  * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unet>`
+    - 8
+    - 4
+    - 4
+    - 4
+    - 4
 
 
 Combined Vessel Dataset (COVD) and Semi-Supervised Learning (SSL)
 =================================================================
 
-COVD-:
-
-.. code-block:: bash
-
-    ### COVD-DRIVE ####
-    dataset=COVD-DRIVE
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIU
-    m2u=M2UNet
-    b_driu=4
-    b_m2u=8
-    # Train
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-STARE ####
-    dataset=COVD-STARE
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIU
-    m2u=M2UNet
-    b_driu=4
-    b_m2u=4
-    # Train
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-IOSTAR ####
-    dataset=COVD-IOSTARVESSEL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIU
-    m2u=M2UNet
-    b_driu=2
-    b_m2u=4
-    # Train
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-CHASEDB1 ####
-    dataset=COVD-CHASEDB1
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIU
-    m2u=M2UNet
-    b_driu=2
-    b_m2u=4
-    # Train
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-HRF ####
-    dataset=COVD-HRF
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIU
-    m2u=M2UNet
-    b_driu=2
-    b_m2u=4
-    # Train
-    bob binseg train $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg train $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-
-COVD-SSL:
-
-.. code-block:: bash
-
-    ### COVD-DRIVE_SSL ####
-    dataset=COVD-DRIVE_SSL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIUSSL
-    m2u=M2UNetSSL
-    b_driu=4
-    b_m2u=4
-    # Train
-    bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-STARE_SSL ####
-    dataset=COVD-STARE_SSL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIUSSL
-    m2u=M2UNetSSL
-    b_driu=4
-    b_m2u=4
-    # Train
-    bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-IOSTAR_SSL ####
-    dataset=COVD-IOSTARVESSEL_SSL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIUSSL
-    m2u=M2UNetSSL
-    b_driu=1
-    b_m2u=2
-    # Train
-    bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-    ### COVD-CHASEDB1_SSL ####
-    dataset=COVD-CHASEDB1_SSL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIUSSL
-    m2u=M2UNetSSL
-    b_driu=2
-    b_m2u=2
-    # Train
-    bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-
-    ### COVD-HRF_SSL ####
-    dataset=COVD-HRF_SSL
-    output=$outputroot"/"$dataset
-    mkdir -p $output
-    # model configs
-    driu=DRIUSSL
-    m2u=M2UNetSSL
-    b_driu=1
-    b_m2u=2
-    # Train
-    bob binseg ssltrain $driu $dataset -b $b_driu -d cuda -o $output"/"$driu -vv
-    bob binseg ssltrain $m2u $dataset -b $b_m2u -d cuda -o $output"/"$m2u -vv
-
-Using your own configs
-======================
-
-Instead of the default configs you can pass the full path of your
-customized dataset and model config (both in PyTorch format).
-The default configs are stored under ``bob.ip.binseg/bob/ip/binseg/configs/``.
-
-.. code-block:: bash
-
-    bob binseg train /path/to/model/config.py /path/to/dataset/config.py
-
-
-
+The following table describes recommended batch sizes for 24Gb of RAM GPU
+card, for semi-supervised learning of COVD- systems.  Use it like this:
+
+.. code-block:: sh
+
+   # change <model> and <dataset> by one of items bellow
+   $ bob binseg train -vv --ssl <model> <dataset> --batch-size=<see-table> --device="cuda:0"
+
+.. list-table::
+
+  * - **Models / Datasets**
+    - :py:mod:`covd-drive <bob.ip.binseg.configs.datasets.starechasedb1iostarhrf544ssldrive>`
+    - :py:mod:`covd-stare <bob.ip.binseg.configs.datasets.drivechasedb1iostarhrf608sslstare>`
+    - :py:mod:`covd-chasedb1 <bob.ip.binseg.configs.datasets.drivestareiostarhrf960sslchase>`
+    - :py:mod:`covd-iostar-vessel <bob.ip.binseg.configs.datasets.drivestarechasedb1hrf1024ssliostar>`
+    - :py:mod:`covd-hrf <bob.ip.binseg.configs.datasets.drivestarechasedb1iostar1168sslhrf>`
+  * - :py:mod:`driu-ssl <bob.ip.binseg.configs.models.driussl>` / :py:mod:`driu-bn <bob.ip.binseg.configs.models.driubnssl>`
+    - 4
+    - 4
+    - 2
+    - 1
+    - 1
+  * - :py:mod:`m2unet <bob.ip.binseg.configs.models.m2unetssl>`
+    - 4
+    - 4
+    - 2
+    - 2
+    - 2
diff --git a/doc/usage.rst b/doc/usage.rst
index 2c8882e4..f5133af1 100644
--- a/doc/usage.rst
+++ b/doc/usage.rst
@@ -18,17 +18,29 @@ semantic binary segmentation with support for the following activities:
   test data, generate ROC curves or visualize prediction results overlayed on
   the original raw images.
 
-Each application is implemented as a command-line utility, that is configurable
-using :ref:`Bob's extensible configuration framework
-<bob.extension.framework>`.  In essence, each command-line option may be
-provided as a variable with the same name in a Python file.  Each file may
-combine any number of variables that are pertinent to an application.  We
-provide a number of :ref:`preset configuration files <bob.ip.binseg.configs>`
-that can be used in one or more of the activities described above.  Our
-command-line framework allows you to refer to these preset configuration files
-using special names (a.k.a. "resources"), that procure and load these for you
-automatically.  Aside preset configuration files, you may also create your own
-to extend existing baseline experiments.
+Each application is implemented as a :ref:`command-line utility
+<bob.ip.binseg.cli>`, that is configurable using :ref:`Bob's extensible
+configuration framework <bob.extension.framework>`.  In essence, each
+command-line option may be provided as a variable with the same name in a
+Python file.  Each file may combine any number of variables that are pertinent
+to an application.
+
+.. tip::
+
+   For reproducibility, we recommend you stick to configuration files when
+   parameterizing our CLI.  Notice some of the options in the CLI interface
+   (e.g. ``--dataset``) cannot be passed via the actual command-line as it
+   requires a :py:class:`concrete PyTorch dataset instance
+   <torch.utils.data.dataset.Dataset>`.
+
+We provide a number of :ref:`preset configuration files
+<bob.ip.binseg.cli.config.list.all>` that can be used in one or more of the
+activities described in this section.  Our command-line framework allows you to
+refer to these preset configuration files using special names (a.k.a.
+"resources"), that procure and load these for you automatically.  Aside preset
+configuration files, you may also create your own to extend existing baseline
+experiments by :ref:`locally copying <bob.ip.binseg.cli.config.copy>` and
+modifying one of our configuration resources.
 
 
 .. toctree::
diff --git a/setup.py b/setup.py
index 0b3b97be..bef1aec4 100644
--- a/setup.py
+++ b/setup.py
@@ -35,11 +35,10 @@ setup(
             "evalpred = bob.ip.binseg.script.binseg:evalpred",
             "gridtable = bob.ip.binseg.script.binseg:testcheckpoints",
             "predict = bob.ip.binseg.script.binseg:predict",
-            "ssltrain = bob.ip.binseg.script.binseg:ssltrain",
-            "train = bob.ip.binseg.script.binseg:train",
             "test = bob.ip.binseg.script.binseg:test",
             "visualize = bob.ip.binseg.script.binseg:visualize",
             "config = bob.ip.binseg.script.config:config",
+            "train = bob.ip.binseg.script.train:train",
         ],
         # bob train configurations
         "bob.ip.binseg.config": [
-- 
GitLab