From dc288792e47b2ea93410354dc60b5412154eb1e0 Mon Sep 17 00:00:00 2001
From: Tim Laibacher <tim.laibacher@idiap.ch>
Date: Wed, 15 May 2019 14:59:01 +0200
Subject: [PATCH] Add plotcomparison, overviewtable. Change plot style

---
 bob/ip/binseg/configs/models/driuj01.py       | 39 --------------
 bob/ip/binseg/configs/models/driulayerwise.py | 51 -------------------
 bob/ip/binseg/configs/models/m2unetj01.py     | 39 --------------
 bob/ip/binseg/configs/models/resunetj01.py    | 39 --------------
 bob/ip/binseg/engine/inferencer.py            |  6 +--
 bob/ip/binseg/engine/trainer.py               |  2 +-
 bob/ip/binseg/script/binseg.py                | 39 ++++++++++++++
 bob/ip/binseg/utils/metric.py                 |  3 +-
 bob/ip/binseg/utils/pdfcreator.py             | 38 ++++++++++++++
 bob/ip/binseg/utils/plot.py                   |  3 +-
 bob/ip/binseg/utils/rsttable.py               | 28 ++++++++++
 conda/meta.yaml                               |  1 +
 requirements.txt                              |  3 +-
 setup.py                                      | 47 ++++++++++++++---
 14 files changed, 154 insertions(+), 184 deletions(-)
 delete mode 100644 bob/ip/binseg/configs/models/driuj01.py
 delete mode 100644 bob/ip/binseg/configs/models/driulayerwise.py
 delete mode 100644 bob/ip/binseg/configs/models/m2unetj01.py
 delete mode 100644 bob/ip/binseg/configs/models/resunetj01.py
 create mode 100644 bob/ip/binseg/utils/pdfcreator.py
 create mode 100644 bob/ip/binseg/utils/rsttable.py

diff --git a/bob/ip/binseg/configs/models/driuj01.py b/bob/ip/binseg/configs/models/driuj01.py
deleted file mode 100644
index 6da8443f..00000000
--- a/bob/ip/binseg/configs/models/driuj01.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.driu import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [200]
-scheduler_gamma = 0.1
-
-# model
-model = build_driu()
-
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
-                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
-    
-# criterion
-criterion = SoftJaccardBCELogitsLoss(alpha=0.1)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/driulayerwise.py b/bob/ip/binseg/configs/models/driulayerwise.py
deleted file mode 100644
index 390145c5..00000000
--- a/bob/ip/binseg/configs/models/driulayerwise.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.driu import build_driu
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import WeightedBCELogitsLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [150]
-scheduler_gamma = 0.1
-
-# model
-model = build_driu()
-
-# pretrained backbone
-pretrained_backbone = modelurls['vgg16']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
-                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
-optim = AdaBound(
-    [
-        {"params": model.backbone.parameters(), "lr": 0.0001,"betas":betas, "final_lr":0.01, "gamma":gamma, "eps" : eps},
-        {"params": model.head.parameters(), "lr": 0.001,"betas":betas, "final_lr":0.1, "gamma":gamma, "eps" : eps},
-    ],
-    betas=betas
-    ,final_lr=final_lr
-    ,gamma=gamma
-    ,eps=eps
-    ,weight_decay=weight_decay
-    ,amsbound=amsbound
-    ,lr=0.00001
-)
-# criterion
-criterion = WeightedBCELogitsLoss(reduction='mean')
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/m2unetj01.py b/bob/ip/binseg/configs/models/m2unetj01.py
deleted file mode 100644
index d17fc4b1..00000000
--- a/bob/ip/binseg/configs/models/m2unetj01.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.m2u import build_m2unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [200]
-scheduler_gamma = 0.1
-
-# model
-model = build_m2unet()
-
-# pretrained backbone
-pretrained_backbone = modelurls['mobilenetv2']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
-                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
-    
-# criterion
-criterion = SoftJaccardBCELogitsLoss(alpha=0.1)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/configs/models/resunetj01.py b/bob/ip/binseg/configs/models/resunetj01.py
deleted file mode 100644
index b9c5093f..00000000
--- a/bob/ip/binseg/configs/models/resunetj01.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from torch.optim.lr_scheduler import MultiStepLR
-from bob.ip.binseg.modeling.resunet import build_res50unet
-import torch.optim as optim
-from torch.nn import BCEWithLogitsLoss
-from bob.ip.binseg.utils.model_zoo import modelurls
-from bob.ip.binseg.modeling.losses import SoftJaccardBCELogitsLoss
-from bob.ip.binseg.engine.adabound import AdaBound
-
-##### Config #####
-lr = 0.001
-betas = (0.9, 0.999)
-eps = 1e-08
-weight_decay = 0
-final_lr = 0.1
-gamma = 1e-3
-eps = 1e-8
-amsbound = False
-
-scheduler_milestones = [200]
-scheduler_gamma = 0.1
-
-# model
-model = build_res50unet()
-
-# pretrained backbone
-pretrained_backbone = modelurls['resnet50']
-
-# optimizer
-optimizer = AdaBound(model.parameters(), lr=lr, betas=betas, final_lr=final_lr, gamma=gamma,
-                 eps=eps, weight_decay=weight_decay, amsbound=amsbound) 
-    
-# criterion
-criterion = SoftJaccardBCELogitsLoss(alpha=0.1)
-
-# scheduler
-scheduler = MultiStepLR(optimizer, milestones=scheduler_milestones, gamma=scheduler_gamma)
diff --git a/bob/ip/binseg/engine/inferencer.py b/bob/ip/binseg/engine/inferencer.py
index bfdf9c1b..c3d9c594 100644
--- a/bob/ip/binseg/engine/inferencer.py
+++ b/bob/ip/binseg/engine/inferencer.py
@@ -193,11 +193,11 @@ def do_inference(
     logger.info("Saving average over all input images: {}".format(metrics_file))
     
     avg_metrics = df_metrics.groupby('threshold').mean()
-    avg_metrics.to_csv(metrics_path)
 
-    avg_metrics["f1_score"] =  2* avg_metrics["precision"]*avg_metrics["recall"]/ \
+    avg_metrics["f1_score"] =  (2* avg_metrics["precision"]*avg_metrics["recall"])/ \
         (avg_metrics["precision"]+avg_metrics["recall"])
     
+    avg_metrics.to_csv(metrics_path)
     maxf1 = avg_metrics['f1_score'].max()
     optimal_f1_threshold = avg_metrics['f1_score'].idxmax()
     
@@ -207,7 +207,7 @@ def do_inference(
     np_avg_metrics = avg_metrics.to_numpy().T
     fig_name = "precision_recall.pdf"
     logger.info("saving {}".format(fig_name))
-    fig = precision_recall_f1iso([np_avg_metrics[0]],[np_avg_metrics[1]], [model.name,None])
+    fig = precision_recall_f1iso([np_avg_metrics[0]],[np_avg_metrics[1]], [model.name,None], title=output_folder)
     fig_filename = os.path.join(results_subfolder, fig_name)
     fig.savefig(fig_filename)
     
diff --git a/bob/ip/binseg/engine/trainer.py b/bob/ip/binseg/engine/trainer.py
index d703ca30..03746e4f 100644
--- a/bob/ip/binseg/engine/trainer.py
+++ b/bob/ip/binseg/engine/trainer.py
@@ -141,6 +141,6 @@ def do_train(
         
     log_plot_file = os.path.join(output_folder,"{}_trainlog.pdf".format(model.name))
     logdf = pd.read_csv(os.path.join(output_folder,"{}_trainlog.csv".format(model.name)),header=None, names=["avg. loss", "median loss","lr","max memory"])
-    fig = loss_curve(logdf)
+    fig = loss_curve(logdf,output_folder)
     logger.info("saving {}".format(log_plot_file))
     fig.savefig(log_plot_file)
diff --git a/bob/ip/binseg/script/binseg.py b/bob/ip/binseg/script/binseg.py
index 7adedbae..e821930d 100644
--- a/bob/ip/binseg/script/binseg.py
+++ b/bob/ip/binseg/script/binseg.py
@@ -27,6 +27,8 @@ from bob.ip.binseg.engine.trainer import do_train
 from bob.ip.binseg.engine.inferencer import do_inference
 from bob.ip.binseg.utils.plot import plot_overview
 from bob.ip.binseg.utils.click import OptionEatAll
+from bob.ip.binseg.utils.pdfcreator import create_pdf, get_paths
+from bob.ip.binseg.utils.rsttable import create_overview_grid
 
 logger = logging.getLogger(__name__)
 
@@ -306,3 +308,40 @@ def compare(output_path_list, output_path, **kwargs):
     fig_filename = os.path.join(output_path, 'precision_recall_comparison.pdf')
     logger.info('saving {}'.format(fig_filename))
     fig.savefig(fig_filename)
+
+
+# Plot overviews
+@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
+@click.option(
+    '--output-path',
+    '-o',
+    required=True,
+    )
+@verbosity_option(cls=ResourceOption)
+def pdfoverview(output_path, **kwargs):
+    """ Creates an overview pdf with all precision vs recall curves present in the output directory.
+    Requires pdflatex to be available on the host."""
+    # PR curves
+    pr_filename = "precision_recall_comparison.pdf"
+    pr_filenames = get_paths(output_path,pr_filename)
+    create_pdf(output_path, pr_filenames, title='Precision vs Recall', tex_filename='pr_overview.tex')
+    
+    # Training curves
+    trainlog_filename = "*trainlog.pdf"
+    tl_file_names = get_paths(output_path,trainlog_filename)
+    create_pdf(output_path, tl_file_names, title='Training', tex_filename='training_overview.tex')
+
+
+# Create grid table with results
+@binseg.command(entry_point_group='bob.ip.binseg.config', cls=ConfigCommand)
+@click.option(
+    '--output-path',
+    '-o',
+    required=True,
+    )
+@verbosity_option(cls=ResourceOption)
+def gridtable(output_path, **kwargs):
+    """ Creates an overview table in grid rst format for all Metrics.csv in the output_path
+    tree structure: ``outputpath/DATABASE/MODEL`` """
+    create_overview_grid(output_path)
+    
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/metric.py b/bob/ip/binseg/utils/metric.py
index b14b9759..d1f5ec63 100644
--- a/bob/ip/binseg/utils/metric.py
+++ b/bob/ip/binseg/utils/metric.py
@@ -3,7 +3,6 @@
 
 from collections import defaultdict
 from collections import deque
-
 import torch
 
 
@@ -62,4 +61,4 @@ def base_metrics(tp, fp, tn, fn):
     jaccard = tp / (tp+fp+fn + ( (tp+fp+fn) == 0) )
     f1_score = (2.0 * tp ) / (2.0 * tp + fp + fn + ( (2.0 * tp + fp + fn) == 0) )
 
-    return [precision, recall, specificity, accuracy, jaccard, f1_score]
+    return [precision, recall, specificity, accuracy, jaccard, f1_score]
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/pdfcreator.py b/bob/ip/binseg/utils/pdfcreator.py
new file mode 100644
index 00000000..fdd1acd3
--- /dev/null
+++ b/bob/ip/binseg/utils/pdfcreator.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from pathlib import Path
+import os
+
+
+def get_paths(output_path, filename):
+    """
+    Parameters
+    ----------
+    output_path : str
+        path in which to look for files
+    filename : str
+
+    Returns
+    -------
+    list 
+        list of file paths
+    """
+    datadir = Path(output_path)
+    file_paths = sorted(list(datadir.glob('**/{}'.format(filename))))
+    file_paths = [f.as_posix() for f in file_paths]
+    return file_paths
+
+
+def create_pdf(output_path, file_paths, title, tex_filename):
+    # setup tex doc
+    textitle = "\\section*{{{}}} \n".format(title, 42)
+    texinit = "\\documentclass{{article}} \\usepackage[utf8]{{inputenc}} \\usepackage[a4paper, margin=2cm]{{geometry}} \\usepackage{{graphicx}} \\begin{{document}} \n".format(42)
+    texclose = "\\end{{document}} \n".format(42)
+    with open (os.path.join(output_path,tex_filename), "w+") as outfile:
+        outfile.write(texinit)
+        outfile.write(textitle)
+        for f in file_paths:
+            outfile.write("\\includegraphics[width=0.5\\textwidth]{{{}}} \n".format(f,42))
+        outfile.write(texclose)
+    # create pdf
+    os.system("pdflatex -output-directory {} {}".format(output_path, os.path.join(output_path,tex_filename)))
\ No newline at end of file
diff --git a/bob/ip/binseg/utils/plot.py b/bob/ip/binseg/utils/plot.py
index 5e6fa29d..0f1bb882 100644
--- a/bob/ip/binseg/utils/plot.py
+++ b/bob/ip/binseg/utils/plot.py
@@ -102,7 +102,7 @@ def precision_recall_f1iso(precision, recall, names, title=None, human_perf_bsds
 
 
 
-def loss_curve(df):
+def loss_curve(df, title):
     ''' Creates a loss curve
     Dataframe with column names:
     ["avg. loss", "median loss","lr","max memory"]
@@ -118,6 +118,7 @@ def loss_curve(df):
     matplotlib.use('agg')
     import matplotlib.pyplot as plt 
     ax1 = df.plot(y="median loss", grid=True)
+    ax1.set_title(title)
     ax1.set_ylabel('median loss')
     ax1.grid(linestyle='--', linewidth=1, color='gray', alpha=0.2)
     ax2 = df['lr'].plot(secondary_y=True,legend=True,grid=True,)
diff --git a/bob/ip/binseg/utils/rsttable.py b/bob/ip/binseg/utils/rsttable.py
new file mode 100644
index 00000000..15a0b68d
--- /dev/null
+++ b/bob/ip/binseg/utils/rsttable.py
@@ -0,0 +1,28 @@
+from bob.ip.binseg.utils.pdfcreator import get_paths
+import pandas as pd
+from tabulate import tabulate
+import os
+
+def create_overview_grid(output_path):
+    """ Reads all Metrics.csv in a certain output path and pivots them to a rst grid table"""
+    filename = 'Metrics.csv'
+    metrics = get_paths(output_path,filename)
+    f1s = []
+    models = []
+    databases = []
+    for m in metrics:
+        metrics = pd.read_csv(m)
+        maxf1 = metrics['f1_score'].max()
+        f1s.append(maxf1)
+        model = m.split('/')[-3]
+        models.append(model)
+        database = m.split('/')[-4]
+        databases.append(database)
+    df = pd.DataFrame()
+    df['database'] = databases
+    df['model'] = models
+    df['f1'] = f1s
+    pivot = df.pivot(index='database',columns='model',values='f1')
+
+    with open (os.path.join(output_path,'Metrics_overview.rst'), "w+") as outfile:
+        outfile.write(tabulate(pivot,headers=pivot.columns, tablefmt="grid"))
\ No newline at end of file
diff --git a/conda/meta.yaml b/conda/meta.yaml
index beff5e34..03070176 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -49,6 +49,7 @@ requirements:
     - bob.db.drishtigs1
     - bob.db.refuge
     - bob.db.iostar
+    - tabulate
     # place other runtime dependencies here (same as requirements.txt)
 
 test:
diff --git a/requirements.txt b/requirements.txt
index d2f0e445..4d442928 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,4 +14,5 @@ torch
 torchvision
 pandas
 matplotlib
-tqdm
\ No newline at end of file
+tqdm
+tabulate
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 19649eff..ac1795b8 100644
--- a/setup.py
+++ b/setup.py
@@ -50,25 +50,56 @@ setup(
           'test = bob.ip.binseg.script.binseg:test',
           'compare =  bob.bin.binseg.script.binseg:compare',
           'testcheckpoints = bob.ip.binseg.script.binseg:testcheckpoints',
+          'pdfoverview = bob.ip.binseg.script.binseg:testcheckpoints',
+          'gridtable = bob.ip.binseg.script.binseg:testcheckpoints',
         ],
 
          #bob hed train configurations
         'bob.ip.binseg.config': [
           'DRIU = bob.ip.binseg.configs.models.driu',
-          'DRIUJ01 = bob.ip.binseg.configs.models.driuj01',
-          'DRIUPAPER = bob.ip.binseg.configs.models.driupaper',
+          'DRIUJ = bob.ip.binseg.configs.models.driuj',
+          'DRIUJ7 = bob.ip.binseg.configs.models.driuj7',
+          'DRIUODJ = bob.ip.binseg.configs.models.driuodj',
           'HED = bob.ip.binseg.configs.models.hed',
+          'HEDJ = bob.ip.binseg.configs.models.hedj',
+          'HEDJ7 = bob.ip.binseg.configs.models.hedj7',
+          'HEDODJ = bob.ip.binseg.configs.models.hedodj',
           'M2UNet = bob.ip.binseg.configs.models.m2unet',
-          'M2UNetJ01 = bob.ip.binseg.configs.models.m2unetj01',
+          'M2UNetJ = bob.ip.binseg.configs.models.m2unetj',
+          'M2UNetJ7 = bob.ip.binseg.configs.models.m2unetj7',
           'UNet = bob.ip.binseg.configs.models.unet',
-          'UNetJ01 = bob.ip.binseg.configs.models.unetj01',
+          'UNetJ = bob.ip.binseg.configs.models.unetj',
+          'UNetJ7 = bob.ip.binseg.configs.models.unetj7',
+          'UNetODJ = bob.ip.binseg.configs.models.unetodj',
           'ResUNet = bob.ip.binseg.configs.models.resunet',
-          'ResUNetJ01 = bob.ip.binseg.configs.models.resunetj01',
+          'ResUNetJ = bob.ip.binseg.configs.models.resunetj',
           'ShapeResUNet = bob.ip.binseg.configs.models.shaperesunet',
-          'DRIVETRAIN = bob.ip.binseg.configs.datasets.drivetrain',
-          'DRIVECROPTRAIN = bob.ip.binseg.configs.datasets.drivecroptrain',
-          'DRIVECROPTEST = bob.ip.binseg.configs.datasets.drivecroptest',
+          'CHASEDB1 = bob.ip.binseg.configs.datasets.chasedb1',
+          'CHASEDB1TEST = bob.ip.binseg.configs.datasets.chasedb1test',
+          'DRIONSDB = bob.ip.binseg.configs.datasets.drionsdb',
+          'DRIONSDBTEST = bob.ip.binseg.configs.datasets.drionsdbtest',
+          'DRISHTIGS1OD = bob.ip.binseg.configs.datasets.dristhigs1od',
+          'DRISHTIGS1ODTEST = bob.ip.binseg.configs.datasets.dristhigs1odtest',
+          'DRISHTIGS1CUP = bob.ip.binseg.configs.datasets.dristhigs1cup',
+          'DRISHTIGS1CUPTEST = bob.ip.binseg.configs.datasets.dristhigs1cuptest',
+          'DRIVE = bob.ip.binseg.configs.datasets.drive',
           'DRIVETEST = bob.ip.binseg.configs.datasets.drivetest',
+          'HRF = bob.ip.binseg.configs.datasets.hrf',
+          'HRFTEST = bob.ip.binseg.configs.datasets.hrftest',
+          'IOSTAROD = bob.ip.binseg.configs.datasets.iostarod',
+          'IOSTARODTEST = bob.ip.binseg.configs.datasets.iostarodtest',
+          'IOSTARVESSEL = bob.ip.binseg.configs.datasets.iostarvessel',
+          'IOSTARVESSELTEST = bob.ip.binseg.configs.datasets.iostarvesseltest',
+          'REFUGECUP = bob.ip.binseg.configs.datasets.refugecup',
+          'REFUGECUPTEST = bob.ip.binseg.configs.datasets.refugecuptest',
+          'REFUGEOD = bob.ip.binseg.configs.datasets.refugeod',
+          'REFUGEODTEST = bob.ip.binseg.configs.datasets.refugeodtest',
+          'RIMONER3CUP = bob.ip.binseg.configs.datasets.rimoner3cup',
+          'RIMONER3CUPTEST = bob.ip.binseg.configs.datasets.rimoner3cuptest',
+          'RIMONER3OD = bob.ip.binseg.configs.datasets.rimoner3od',
+          'RIMONER3ODTEST = bob.ip.binseg.configs.datasets.rimoner3odtest',
+          'STARE = bob.ip.binseg.configs.datasets.stare',
+          'STARETEST = bob.ip.binseg.configs.datasets.staretest',
           ]
     },
 
-- 
GitLab