Update docs and add DRIUBN

bob/ip/binseg/configs/models/driubn.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.driubn import build_driu
+import torch.optim as optim
+from torch.nn import BCEWithLogitsLoss
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_driu()
+
+# pretrained backbone
+pretrained_backbone = modelurls['vgg16_bn']
+
+# optimizer
+optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
+                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
+# criterion
+criterion = SoftJaccardBCELogitsLoss(alpha=0.7)
+
+# scheduler
+scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)

bob/ip/binseg/configs/models/driubnssl.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from torch.optim.lr_scheduler import MultiStepLR
+from bob.ip.binseg.modeling.driubn import build_driu
+import torch.optim as optim
+from torch.nn import BCEWithLogitsLoss
+from bob.ip.binseg.utils.model_zoo import modelurls
+from bob.ip.binseg.modeling.losses import MixJacLoss
+from bob.ip.binseg.engine.adabound import AdaBound
+
+##### Config #####
+lr = 0.001
+betas = (0.9, 0.999)
+eps = 1e-08
+weight_decay = 0
+final_lr = 0.1
+gamma = 1e-3
+eps = 1e-8
+amsbound = False
+
+scheduler_milestones = [900]
+scheduler_gamma = 0.1
+
+# model
+model = build_driu()
+
+# pretrained backbone
+pretrained_backbone = modelurls['vgg16_bn']
+
+# optimizer
+optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
+                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
+
+# criterion
+criterion = MixJacLoss(lambda_u=0.05, jacalpha=0.7)
+
+# scheduler
+scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)

bob/ip/binseg/engine/inferencer.py

@@ -192,10 +192,13 @@ def do_inference(
     logger.info("Saving average over all input images: {}".format(metrics_file))
     
     avg_metrics = df_metrics.groupby('threshold').mean()
+    std_metrics = df_metrics.groupby('threshold').std()
 
     avg_metrics["f1_score"] =  (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
         (avg_metrics["precision"]+avg_metrics["recall"])
     
+    avg_metrics["std_f1"] = std_metrics["f1_score"]
+    
     avg_metrics.to_csv(metrics_path)
     maxf1 = avg_metrics['f1_score'].max()
     optimal_f1_threshold = avg_metrics['f1_score'].idxmax()

bob/ip/binseg/modeling/driubn.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import torch
+from torch import nn
+from collections import OrderedDict
+from bob.ip.binseg.modeling.backbones.vgg import vgg16_bn
+from bob.ip.binseg.modeling.make_layers import conv_with_kaiming_uniform,convtrans_with_kaiming_uniform, UpsampleCropBlock
+
+class ConcatFuseBlock(nn.Module):
+    """ 
+    Takes in four feature maps with 16 channels each, concatenates them 
+    and applies a 1x1 convolution with 1 output channel. 
+    """
+    def __init__(self):
+        super().__init__()
+        self.conv = nn.Sequential(
+            conv_with_kaiming_uniform(4*16,1,1,1,0)
+            ,nn.BatchNorm2d(1)
+        )
+    def forward(self,x1,x2,x3,x4):
+        x_cat = torch.cat([x1,x2,x3,x4],dim=1)
+        x = self.conv(x_cat)
+        return x 
+            
+class DRIU(nn.Module):
+    """
+    DRIU head module
+    Based on paper by `Maninis et al. (2016)`_ 
+    Parameters
+    ----------
+    in_channels_list : list
+        number of channels for each feature map that is returned from backbone
+    """
+    def __init__(self, in_channels_list=None):
+        super(DRIU, self).__init__()
+        in_conv_1_2_16, in_upsample2, in_upsample_4, in_upsample_8 = in_channels_list
+
+        self.conv1_2_16 = nn.Conv2d(in_conv_1_2_16, 16, 3, 1, 1)
+        # Upsample layers
+        self.upsample2 = UpsampleCropBlock(in_upsample2, 16, 4, 2, 0)
+        self.upsample4 = UpsampleCropBlock(in_upsample_4, 16, 8, 4, 0)
+        self.upsample8 = UpsampleCropBlock(in_upsample_8, 16, 16, 8, 0)
+        
+        # Concat and Fuse
+        self.concatfuse = ConcatFuseBlock()
+
+    def forward(self,x):
+        """
+        Parameters
+        ----------
+        x : list
+            list of tensors as returned from the backbone network.
+            First element: height and width of input image. 
+            Remaining elements: feature maps for each feature level.
+
+        Returns
+        -------
+        :py:class:`torch.Tensor`
+        """
+        hw = x[0]
+        conv1_2_16 = self.conv1_2_16(x[1])  # conv1_2_16   
+        upsample2 = self.upsample2(x[2], hw) # side-multi2-up
+        upsample4 = self.upsample4(x[3], hw) # side-multi3-up
+        upsample8 = self.upsample8(x[4], hw) # side-multi4-up
+        out = self.concatfuse(conv1_2_16, upsample2, upsample4, upsample8)
+        return out
+
+def build_driu():
+    """ 
+    Adds backbone and head together
+
+    Returns
+    -------
+    :py:class:torch.nn.Module
+    """
+    backbone = vgg16_bn(pretrained=False, return_features = [3, 8, 14, 22])
+    driu_head = DRIU([64, 128, 256, 512])
+
+    model = nn.Sequential(OrderedDict([("backbone", backbone), ("head", driu_head)]))
+    model.name = "DRIUBN"
+    return model
\ No newline at end of file

bob/ip/binseg/script/binseg.py

@@ -113,6 +113,14 @@ def binseg():
     required=True,
     default='cpu',
     cls=ResourceOption)
+@click.option(
+    '--seed',
+    '-s',
+    help='torch random seed',
+    show_default=True,
+    required=False,
+    default=42,
+    cls=ResourceOption)
 
 @verbosity_option(cls=ResourceOption)
 def train(model
@@ -126,11 +134,12 @@ def train(model
         ,dataset
         ,checkpoint_period
         ,device
+        ,seed
         ,**kwargs):
     """ Train a model """
     
     if not os.path.exists(output_path): os.makedirs(output_path)
-    
+    torch.manual_seed(seed)
     # PyTorch dataloader
     data_loader = DataLoader(
         dataset = dataset
@@ -481,6 +490,14 @@ def visualize(dataset, output_path, **kwargs):
     required=True,
     default='900',
     cls=ResourceOption)
+@click.option(
+    '--seed',
+    '-s',
+    help='torch random seed',
+    show_default=True,
+    required=False,
+    default=42,
+    cls=ResourceOption)
 
 @verbosity_option(cls=ResourceOption)
 def ssltrain(model
@@ -495,11 +512,12 @@ def ssltrain(model
         ,checkpoint_period
         ,device
         ,rampup
+        ,seed
         ,**kwargs):
     """ Train a model """
     
     if not os.path.exists(output_path): os.makedirs(output_path)
-    
+    torch.manual_seed(seed)
     # PyTorch dataloader
     data_loader = DataLoader(
         dataset = dataset

bob/ip/binseg/utils/plot.py

@@ -190,7 +190,7 @@ def plot_overview(outputfolders,title):
           rows = outfile.readlines()
           lastrow = rows[-1]
           parameter = int(lastrow.split()[1].replace(',',''))
-        name = '[P={:.2f}M] {} {}'.format(parameter/100**3, modelname, datasetname)
+        name = '[P={:.2f}M] {} {}'.format(parameter/100**3, modelname, "")
         names.append(name)
     #title = folder.split('/')[-4]
     fig = precision_recall_f1iso(precisions,recalls,names,title)

doc/benchmarkresults.rst

@@ -6,20 +6,62 @@
 Benchmark Results
 ==================
 
-Dice Scores
+F1 Scores
 ===========
 
 * Benchmark results for models: DRIU, HED, M2UNet and U-Net.
-* Train-Test split as indicated in :ref:`bob.ip.binseg.datasets`
-
-+--------+----------+--------+---------+--------+--------+
-|        | CHASEDB1 | DRIVE  | HRF1168 | IOSTAR | STARE  |
-+--------+----------+--------+---------+--------+--------+
-| DRIU   | 0.8114   | 0.8226 | 0.7865  | 0.8273 | 0.8286 |
-+--------+----------+--------+---------+--------+--------+
-| HED    | 0.8111   | 0.8192 | 0.7868  | 0.8275 | 0.8250 |
-+--------+----------+--------+---------+--------+--------+
-| M2UNet | 0.8035   | 0.8051 | 0.7838  | 0.8194 | 0.8174 |
-+--------+----------+--------+---------+--------+--------+
-| UNet   | 0.8136   | 0.8237 | 0.7941  | 0.8203 | 0.8306 |
-+--------+----------+--------+---------+--------+--------+
+* Models are trained and tested on the same dataset using the train-test split as indicated in :ref:`bob.ip.binseg.datasets`
+* standard-deviations across all test images are indicated in brakets
+
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+| F1 (std) | CHASEDB1        | DRIVE           | HRF1168         | IOSTAR          | STARE           |
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+| DRIU     | 0.8114 (0.0206) | 0.8226 (0.0142) | 0.7865 (0.0545) | 0.8273 (0.0199) | 0.8286 (0.0368) |
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+| HED      | 0.8111 (0.0214) | 0.8192 (0.0136) | 0.7868 (0.0576) | 0.8275 (0.0201) | 0.8250 (0.0375) |
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+| M2UNet   | 0.8035 (0.0195) | 0.8051 (0.0141) | 0.7838 (0.0572) | 0.8194 (0.0201) | 0.8174 (0.0409) |
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+| UNet     | 0.8136 (0.0209) | 0.8237 (0.0145) | 0.7914 (0.0516) | 0.8203 (0.0190) | 0.8306 (0.0421) |
++----------+-----------------+-----------------+-----------------+-----------------+-----------------+
+
+
+.. figure:: img/pr_CHASEDB1.png
+   :scale: 30 %
+   :align: center
+   :alt: model comparisons
+
+   CHASE_DB1: Precision vs Recall curve, F1 scores and
+   number of parameter of each model.
+
+.. figure:: img/pr_DRIVE.png
+   :scale: 30 %
+   :align: center
+   :alt: model comparisons
+
+   DRIVE: Precision vs Recall curve, F1 scores and
+   number of parameter of each model.
+
+.. figure:: img/pr_HRF.png
+   :scale: 30 %
+   :align: center
+   :alt: model comparisons
+
+   HRF: Precision vs Recall curve, F1 scores and
+   number of parameter of each model.
+
+.. figure:: img/pr_IOSTARVESSEL.png
+   :scale: 30 %
+   :align: center
+   :alt: model comparisons
+
+   IOSTAR: Precision vs Recall curve, F1 scores and
+   number of parameter of each model.
+
+.. figure:: img/pr_STARE.png
+   :scale: 30 %
+   :align: center
+   :alt: model comparisons
+
+   STARE: Precision vs Recall curve, F1 scores and
+   number of parameter of each model.

doc/covdresults.rst

@@ -6,7 +6,7 @@
 COVD- and COVD-SLL Results
 ==========================
 
-Dice Scores
+F1 Scores
 ===========
 
 +-------------------+---------------+---------+

setup.py

@@ -58,7 +58,9 @@ setup(
          #bob train configurations
         'bob.ip.binseg.config': [
           'DRIU = bob.ip.binseg.configs.models.driu',
+          'DRIUBN = bob.ip.binseg.configs.models.driubn',
           'DRIUSSL = bob.ip.binseg.configs.models.driussl',
+          'DRIUBNSSL = bob.ip.binseg.configs.models.driubnssl',
           'DRIUOD = bob.ip.binseg.configs.models.driuod',
           'HED = bob.ip.binseg.configs.models.hed',
           'M2UNet = bob.ip.binseg.configs.models.m2unet',