Add the multimodal lab

00_section_example.md

-# Section 1
-
-## Slide 1
-This is the first slide
-
-## Slide 2
-This is the second slide
-

Makefile

-TARGET := lab.pdf
-SECTIONS := $(sort $(wildcard ??_section*.md))
-
-# builds a single section
-%.pdf: metadata.yaml %.md
-	pandoc -H header.tex -f markdown -t beamer -o $@ $?
-
-# builds the whole lecture
-$(TARGET): metadata.yaml $(SECTIONS)
-	pandoc -H header.tex -f markdown -t beamer -o $@ $^
-
-all: $(TARGET)
-
-.PHONY: clean
-
-clean:
-	@rm -vf $(shell find . -name "*~")
-	@rm -vf $(TARGET) $(patsubst %.md,%.pdf,$(SECTIONS))

README.md

-# Biometrics Lab
-
-## Description
-
-Slides of the lab:
-- [ ] lab 1: face,
-- [ ] lab 2: iris and fingerprint,
-- [ ] lab 3: speaker,
-- [x] lab 4: vein and multi-modal,
-- [ ] lab 5: spoofing.
-
-## Lab material
-
-[Lab material](./lab/lab.md)
-
-## LaTeX Beamer slides with pandoc
-
-We are using [pandoc](https://pandoc.org) to convert simple .md files into latex beamer.
-
-Please refer to the lecture example (git@gitlab.idiap.ch:master-biometrics/lecture-example.git) for more details.
-
-## Setting up
-
-**You need to refer** to the same section the lecture example (git@gitlab.idiap.ch:master-biometrics/lecture-example.git) for more details.
-
-For a quick setup just do
-```sh
-$ git clone git@gitlab.idiap.ch:master-biometrics/graphics.git
-$ git clone git@gitlab.idiap.ch:master-biometrics/<this-lab>.git
-$ cd <this-lab>
-$ rm -rf graphics
-$ ln -s ../graphics .
-```
-
-## Building the slides
-Proceed in 2 steps when working on the slides in .md files:
-1. edit only the .md files
-2. check if it compiles
-
-To compile
-* You can just compile the slides of one md file:
-
-```sh
-$ make 01_section1.pdf
-```
-***notice the name of the make target is the filename with a `.pdf` extension***
-
-* Or your can compile all the slides (`lab.pdf`) that merges all sections into it. 
-
-```sh
-$ make
-```
-***The input `.md` files are sorted before inclusion, so ensure that the name sorting works fine before building the whole document.***
-
-* You can use flag `-B` with make to force recompilation if necessary. Otherwise, our Makefile is set to detect changes automatically.
-
-```sh
-$ make -B
-```
-It results in a ``lab.pdf`` file. 
-**Please don't git add/commit/push this file nor any other PDF by-product. They are git-ignored by default.**
+# Lab on score-level fusion in multimodal biometrics
 
+To run this lab, download this
+[tarball](https://gitlab.idiap.ch/biometric-resources/lab-multimodal/-/archive/master/lab-multimodal-master.tar.bz2)
+and extract it. Then, double click on ``fusion_lab.ipynb`` to open the
+exercise.

beamerthemeidiapbsp.sty

-\mode<presentation>
-
-% Requirement
-\RequirePackage{tikz}
-\RequirePackage{graphicx}
-%\RequirePackage{fontspec}
-\RequirePackage{hyperref}
-
-% Colors
-\definecolor{unidistancedark}{RGB}{24,70,139}
-\definecolor{unidistancelite}{RGB}{20,118,198}
-\definecolor{darkred}{RGB}{180,0,0}
-\definecolor{darkgreen}{RGB}{0,128,0}
-
-% Defines the background of all slides
-\setbeamertemplate{background}{
-%  \begin{tikzpicture}
-%    \fill[color=unidistancedark] (0,2.5) rectangle (2.5,5);
-%    \fill[color=unidistancelite] (2.7,2.5) rectangle (\the\paperwidth,5);
-%    \node (logo) at (0,8.3)
-%      {\includegraphics[width=.5\paperwidth]{graphics/logo/idiap}};
-%    \ifnum\thepage>1\relax%
-%    \fill[white,opacity=1] (0,0)
-%    rectangle(\the\paperwidth,\the\paperheight);
-%    \fi
-%  \end{tikzpicture}
-  \begin{tikzpicture}
-    % The x-coordinate of the image should NOT be 0 -- it is the center of the image
-    \node (logo) at (2,7.5) {\includegraphics[width=.25\paperwidth]{graphics/logo/idiap}};
-    \fill[color=unidistancedark] (0,3.0) rectangle (2.5,5);
-    \fill[color=unidistancelite] (2.7,3.0) rectangle (\the\paperwidth,5);
-    %\fill[color=white] (0,8.3) rectangle (2.5,5);
-    %\fill[color=unidistancedark] (0,2.5) rectangle (2.5,5);
-    %\fill[color=unidistancelite] (2.7,2.5) rectangle (\the\paperwidth,5);
-    \ifnum\thepage>1\relax%
-    \fill[white,opacity=1] (0,0)
-    rectangle(\the\paperwidth,\the\paperheight);
-    \fi
-  \end{tikzpicture}
-}
-
-% Defines some fonts and colors
-\usefonttheme{structurebold}
-\setbeamercolor{unidistance title}{fg=white}
-%\setbeamercolor{unidistance title}{fg=unidistancedark}
-\setbeamerfont{unidistance title}{series=\bfseries,size=\large}
-\setbeamercolor{unidistance subtitle}{fg=white}
-%\setbeamercolor{unidistance subtitle}{fg=unidistancedark}
-\setbeamerfont{unidistance subtitle}{series=\bfseries,size=\normalsize}
-\setbeamercolor{unidistance author}{fg=black}
-\setbeamerfont{unidistance author}{series=\bfseries}
-
-% Title page
-\defbeamertemplate*{title page}{unidistance}[1][]
-{
-  % Normally vkip here to overlay with the box in background
-  \vskip3.2cm%
-  \begin{beamercolorbox}[wd=14cm,leftskip=3cm,#1]{title page header}
-    \usebeamerfont{unidistance title}\usebeamercolor[fg]{unidistance title}\inserttitle\par%
-  \end{beamercolorbox}%
-  \vskip0.1cm%
-  \begin{beamercolorbox}[wd=14cm,leftskip=3cm,#1]{subtitle page header}
-    \usebeamerfont{unidistance subtitle}\usebeamercolor[fg]{unidistance subtitle}\insertsubtitle\par%
-  \end{beamercolorbox}
-  \vskip1cm%
-  \begin{beamercolorbox}[wd=14cm,leftskip=3cm,#1]{author}
-    \usebeamerfont{unidistance author}\usebeamercolor[fg]{unidistance author}\insertauthor%
-  \end{beamercolorbox}
-}
-
-%% Defines the frametitle header
-%\defbeamertemplate*{frametitle}{unidistance}[1][]
-%{
-%  \begin{beamercolorbox}[wd=\paperwidth,ht=50pt]{frametitle}
-%
-%    \begin{tikzpicture}
-%      %tikz coordinates are (0,0) on the bottom left of the frame, as defined
-%      %above - the format is (x,y) - (horizontal,vertical) displacements
-%
-%      %use the entire region as a canvas, paint it in white
-%      \useasboundingbox[fill=white](0,0) rectangle(\paperwidth,50pt);
-%
-%      %%put logo, starting on the top-left (n.b.: position w.r.t. image center)
-%      %\node (logo) at (.15\paperwidth+5,25pt)
-%      %  {\includegraphics[width=.1\paperwidth]{graphics/logo/idiap}};
-%
-%      %options: title alone or title + subtitle
-%      \ifx\insertframesubtitle\@empty%
-%      {\node[anchor=west,unidistancedark,font=\LARGE] at (.32\paperwidth,17pt){\insertframetitle};}
-%      \else%
-%      {\node[anchor=west,unidistancedark,font=\Large] at (0.32\paperwidth,28pt){\insertframetitle};%
-%        \node[anchor=west,unidistancelite,font=\small] at (0.32\paperwidth,14pt){\insertframesubtitle};}%
-%      \fi
-%    \end{tikzpicture}
-%
-%  \end{beamercolorbox}
-%}
-
-% Title page
-\defbeamertemplate*{toc}{unidistance}[1][]
-{
-}
-
-% Set itemize and enumeration lists
-\setbeamertemplate{itemize items}[square]
-\setbeamercolor*{itemize item}{fg=unidistancedark}
-\setbeamercolor*{itemize subitem}{fg=unidistancedark}
-\setbeamercolor*{itemize subsubitem}{fg=unidistancedark}
-\setbeamertemplate{enumerate items}[default]
-\setbeamercolor*{enumerate item}{fg=unidistancedark}
-\setbeamercolor*{enumerate subitem}{fg=unidistancedark}
-\setbeamercolor*{enumerate subsubitem}{fg=unidistancedark}
-
-% Configure blocks
-\setbeamertemplate{blocks}[square]
-\setbeamercolor{block title}{bg=unidistancelite,fg=white}
-\setbeamercolor{block body}{bg=unidistancelite!10,fg=black}
-\setbeamercolor{block title alerted}{bg=darkred!80,fg=white}
-\setbeamercolor{block body alerted}{bg=red!10,fg=black}
-\setbeamercolor{block title example}{bg=darkgreen!80,fg=white}
-\setbeamercolor{block body example}{bg=green!10,fg=black}
-
-% Disables navigation symbols
-\setbeamertemplate{navigation symbols}{}
-
-% Defines the frame footer
-\setbeamertemplate{footline}{
-  \begin{beamercolorbox}[wd=\the\paperwidth,ht=20pt]{section in head/foot}%
-    \begin{tikzpicture}
-      %use the entire region as a canvas, paint it in white
-      \useasboundingbox[fill=white](0,0) rectangle(\paperwidth,20pt);
-      %left box, darker color
-      \path[fill=unidistancedark] (30pt,0) rectangle (110pt,20pt);
-      \node[anchor=center,align=center,white,font={\fontsize{5pt}{3}\selectfont}] at (72pt,10pt) { \href{http://www.idiap.ch}{Idiap} };
-      %right box, lighter color with title and page number
-      \path[fill=unidistancelite] (112pt,0) rectangle (\paperwidth-30pt,20pt);
-      \node[anchor=west,white,font={\fontsize{5pt}{3}\selectfont}] at (115pt,10pt) { \insertsubtitle\ | \inserttitle\ };
-      \node[anchor=east,white,font={\fontsize{5pt}{3}\selectfont}] at (\paperwidth-30pt,10pt) { \insertframenumber/\inserttotalframenumber };
-    \end{tikzpicture}
-  \end{beamercolorbox}
-}
-
-% Redefines maketitle so we don't need to put it in a frame
-% User needs to do just "\maketitle" after "\begin{document}"
-\def\maketitle{%
-  \ifbeamer@inframe
-    \titlepage
-  \else
-    \begin{frame}[plain]
-      \titlepage
-    \end{frame}
-  \fi
-}
-
-% Settings
-\setbeamercolor*{title page header}{fg=white}
-\setbeamercolor*{author}{fg=white}
-\setbeamercolor*{date}{fg=white}
-
-% Define custom front page
-\def\makefrontpage{%
-  \begin{frame}[plain,c,noframenumbering]
-    \begin{center}
-      \includegraphics[width=.75\paperwidth]{graphics/logo/idiap}
-      \vskip3em
-      \bfseries\href{http://www.idiap.ch}{www.idiap.ch}
-    \end{center}
-  \end{frame}
-}
-
-% Define custom last page
-\def\makelastpage{%
-  \begin{frame}[plain,c,noframenumbering]
-    \begin{center}
-      \includegraphics[width=.75\paperwidth]{graphics/logo/idiap}
-      \vskip3em
-      \bfseries\href{http://www.idiap.ch}{www.idiap.ch}
-    \end{center}
-  \end{frame}
-}
-
-\mode<all>

fusion.py

+from bob.bio.base.score.load import split
+from bob.measure import eer_threshold
+from IPython.display import IFrame
+from subprocess import check_call
+import os
+
+REGENERATE = False
+
+
+def hist(arguments, name, width=800, height=400, extra_options=['--no-line']):
+    if not name.endswith('.pdf'):
+        name += '.pdf'
+    if REGENERATE or (not os.path.exists(name)):
+        check_call(['bob', 'bio', 'hist'] + arguments + ['-o', name] +
+                   extra_options)
+    return IFrame(name, width=width, height=height)
+
+
+def epc(arguments, name, width=800, height=400, extra_options=[]):
+    if not name.endswith('.pdf'):
+        name += '.pdf'
+    if REGENERATE or (not os.path.exists(name)):
+        check_call(['bob', 'bio', 'epc'] + arguments + ['-o', name] +
+                   extra_options)
+    return IFrame(name, width=width, height=height)
+
+
+def roc(arguments, name, width=800, height=400, extra_options=[]):
+    if not name.endswith('.pdf'):
+        name += '.pdf'
+    if REGENERATE or (not os.path.exists(name)):
+        check_call(['bob', 'bio', 'roc'] + arguments + ['-o', name] +
+                   extra_options)
+    return IFrame(name, width=width, height=height)
+
+
+def boundary(algorithm):
+    """show decision boundary"""
+    output = 'boundary_{}.pdf'.format(algorithm)
+    if REGENERATE or (not os.path.exists(output)):
+        threshold = eer_threshold(*split(
+            'scores/multi-algorithm/{}/standard/scores-dev'.format(algorithm)))
+        check_call([
+            'bin/bob', 'fusion', 'boundary', '-m',
+            'scores/multi-algorithm/{}/standard/Model.pkl'.format(algorithm),
+            '-t',
+            str(threshold), '-o', output, '-g', '100', '--x-label',
+            'System 1 scores', '--y-label', 'System 2 scores',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-eval'
+        ])
+    return IFrame(output, width=800, height=400)
+
+
+def hist_plot(name):
+    if name == 'hist-multi-alg':
+        return hist([
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-eval'
+        ],
+            'hist-multi-alg',
+            extra_options=[
+            '--titles', 'System 1,System 2', '-sp', '211',
+                        '--no-line'
+        ])
+
+    elif name == 'hist-fusion-multi-alg-sum-no-norm':
+        return hist([
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/sum/no_normalize/scores-eval'
+        ],
+            'hist-fusion-multi-alg-sum-no-norm',
+            extra_options=[
+            '--titles', 'System 1,System 2,Sum Fusion (No Norm)',
+                        '-sp', '311', '--no-line'
+        ])
+    elif name == 'hist-multi-alg-z-norm':
+        return hist(
+            [
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/normalized/znrom/system-1/scores-eval',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/normalized/znrom/system-2/scores-eval'
+            ],
+            'hist-multi-alg-z-norm',
+            extra_options=[
+                '--titles',
+                'System 1,System 1 (Z-Norm),System 2,System 2 (Z-Norm)', '-sp',
+                '411', '--no-line'
+            ])
+    elif name == 'hist-fusion-multi-alg-sum-znorm':
+        return hist(
+            [
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/sum/no_normalize/scores-eval',
+                'scores/multi-algorithm/sum/znorm/scores-eval'
+            ],
+            'hist-fusion-multi-alg-sum-znorm',
+            extra_options=[
+                '--titles',
+                'System 1,System 2,Sum Fusion (No Norm),Sum Fusion (Z-Norm)',
+                '-sp', '411', '--no-line'
+            ])
+    elif name == 'hist-multi-alg-all-norm':
+        return hist(
+            [
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/normalized/znrom/system-1/scores-eval',
+                'scores/multi-algorithm/normalized/znrom/system-2/scores-eval',
+                'scores/multi-algorithm/normalized/standard/system-1/scores-eval',
+                'scores/multi-algorithm/normalized/standard/system-2/scores-eval',
+                'scores/multi-algorithm/normalized/tanh/system-1/scores-eval',
+                'scores/multi-algorithm/normalized/tanh/system-2/scores-eval',
+            ],
+            'hist-multi-alg-all-norm',
+            extra_options=[
+                '--titles',
+                'System 1,System 2,System 1 (Z-Norm),System 2 (Z-Norm),System 1 (Standard),System 2 (Standard),System 1 (Tanh),System 2 (Tanh)',
+                '-sp', '421', '--no-line'
+            ])
+
+    else:
+        raise ValueError("Unknown hist plot")
+
+
+def epc_plot(name):
+    if name == 'epc-fusion-multi-alg-sum-no-norm':
+        return epc([
+            'scores/multi-algorithm/system-1-scores-dev',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-dev',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/sum/no_normalize/scores-dev',
+            'scores/multi-algorithm/sum/no_normalize/scores-eval',
+        ],
+            'epc-fusion-multi-alg-sum-no-norm',
+            extra_options=[
+            '--legends', 'System 1,System 2,Sum Fusion (No Norm)',
+            '-S'
+        ])
+    elif name == 'epc-multi-alg-z-norm':
+        return epc([
+            'scores/multi-algorithm/system-1-scores-dev',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-dev',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/normalized/znrom/system-1/scores-dev',
+            'scores/multi-algorithm/normalized/znrom/system-1/scores-eval',
+            'scores/multi-algorithm/normalized/znrom/system-2/scores-dev',
+            'scores/multi-algorithm/normalized/znrom/system-2/scores-eval'
+        ],
+            'epc-multi-alg-z-norm',
+            extra_options=[
+            '--legends',
+            'System 1,System 2,System 1 (Z-Norm),System 2 (Z-Norm)',
+            '-S'
+        ])
+    elif name == 'epc-fusion-multi-alg-sum-z-norm':
+        return epc(
+            [
+                'scores/multi-algorithm/system-1-scores-dev',
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/system-2-scores-dev',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/sum/no_normalize/scores-dev',
+                'scores/multi-algorithm/sum/no_normalize/scores-eval',
+                'scores/multi-algorithm/sum/znorm/scores-dev',
+                'scores/multi-algorithm/sum/znorm/scores-eval'
+            ],
+            'epc-fusion-multi-alg-sum-z-norm',
+            extra_options=[
+                '--legends',
+                'System 1,System 2,Sum Fusion (No Norm),Sum Fusion (Z-Norm)',
+                '-S'
+            ])
+    elif name == 'epc-fusion-multi-alg-sum-all':
+        return epc(
+            [
+                'scores/multi-algorithm/system-1-scores-dev',
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/system-2-scores-dev',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/sum/no_normalize/scores-dev',
+                'scores/multi-algorithm/sum/no_normalize/scores-eval',
+                'scores/multi-algorithm/sum/znorm/scores-dev',
+                'scores/multi-algorithm/sum/znorm/scores-eval',
+                'scores/multi-algorithm/sum/standard/scores-dev',
+                'scores/multi-algorithm/sum/standard/scores-eval',
+                'scores/multi-algorithm/sum/tanh/scores-dev',
+                'scores/multi-algorithm/sum/tanh/scores-eval',
+            ],
+            'epc-fusion-multi-alg-sum-all',
+            extra_options=[
+                '--legends',
+                'System 1,System 2,Sum (No Norm),Sum (ZNorm),Sum (Standard),Sum (Tanh)',
+                '-S'
+            ])
+    elif name == 'epc-fusion-multi-alg-all':
+        return epc([
+            'scores/multi-algorithm/system-1-scores-dev',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-dev',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/sum/standard/scores-dev',
+            'scores/multi-algorithm/sum/standard/scores-eval',
+            'scores/multi-algorithm/gmm/standard/scores-dev',
+            'scores/multi-algorithm/gmm/standard/scores-eval',
+            'scores/multi-algorithm/llr/standard/scores-dev',
+            'scores/multi-algorithm/llr/standard/scores-eval',
+            'scores/multi-algorithm/plr-2/standard/scores-dev',
+            'scores/multi-algorithm/plr-2/standard/scores-eval',
+        ],
+            'epc-fusion-multi-alg-all',
+            extra_options=[
+            '--legends', 'System 1,System 2,Sum,GMM,LLR,PLR-2', '-S'
+        ])
+    elif name == 'epc-fusion-multi-modal-all':
+        return epc([
+            'scores/multi-modal/face/scores-dev',
+            'scores/multi-modal/face/scores-eval',
+            'scores/multi-modal/voice/scores-dev',
+            'scores/multi-modal/voice/scores-eval',
+            'scores/multi-modal/fingerprint/scores-dev',
+            'scores/multi-modal/fingerprint/scores-eval',
+            'scores/multi-modal/sum/standard/scores-dev',
+            'scores/multi-modal/sum/standard/scores-eval',
+            'scores/multi-modal/gmm/standard/scores-dev',
+            'scores/multi-modal/gmm/standard/scores-eval',
+            'scores/multi-modal/llr/standard/scores-dev',
+            'scores/multi-modal/llr/standard/scores-eval',
+            'scores/multi-modal/plr-2/standard/scores-dev',
+            'scores/multi-modal/plr-2/standard/scores-eval',
+        ],
+            'epc-fusion-multi-modal-all',
+            extra_options=[
+            '--legends', 'Face,Voice,Fingerprint,Sum,GMM,LLR,PLR-2',
+            '-S'
+        ])
+    else:
+        raise ValueError("Unknown epc plot")
+
+
+def roc_plot(name):
+    if name == 'roc-fusion-multi-alg-sum-no-norm':
+        return roc([
+            'scores/multi-algorithm/system-1-scores-dev',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-dev',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/sum/no_normalize/scores-dev',
+            'scores/multi-algorithm/sum/no_normalize/scores-eval',
+        ],
+            'roc-fusion-multi-alg-sum-no-norm',
+            extra_options=[
+            '--legends', 'System 1,System 2,Sum Fusion (No Norm)',
+            '-S', '-e', '-la', '1e-2'
+        ])
+    elif name == 'roc-fusion-multi-alg-sum-no-norm':
+        return roc(
+            [
+                'scores/multi-algorithm/system-1-scores-dev',
+                'scores/multi-algorithm/system-1-scores-eval',
+                'scores/multi-algorithm/system-2-scores-dev',
+                'scores/multi-algorithm/system-2-scores-eval',
+                'scores/multi-algorithm/sum/no_normalize/scores-dev',
+                'scores/multi-algorithm/sum/no_normalize/scores-eval',
+                'scores/multi-algorithm/sum/znorm/scores-dev',
+                'scores/multi-algorithm/sum/znorm/scores-eval',
+            ],
+            'roc-fusion-multi-alg-sum-no-norm',
+            extra_options=[
+                '--legends',
+                'System 1,System 2,Sum Fusion (No Norm),Sum Fusion (Z-Norm)',
+                '-S', '-e', '-la', '1e-2'
+            ])
+    elif name == 'roc-fusion-multi-alg-all':
+        return roc([
+            'scores/multi-algorithm/system-1-scores-dev',
+            'scores/multi-algorithm/system-1-scores-eval',
+            'scores/multi-algorithm/system-2-scores-dev',
+            'scores/multi-algorithm/system-2-scores-eval',
+            'scores/multi-algorithm/sum/standard/scores-dev',
+            'scores/multi-algorithm/sum/standard/scores-eval',
+            'scores/multi-algorithm/gmm/standard/scores-dev',
+            'scores/multi-algorithm/gmm/standard/scores-eval',
+            'scores/multi-algorithm/llr/standard/scores-dev',
+            'scores/multi-algorithm/llr/standard/scores-eval',
+            'scores/multi-algorithm/plr-2/standard/scores-dev',
+            'scores/multi-algorithm/plr-2/standard/scores-eval',
+        ],
+            'roc-fusion-multi-alg-all',
+            extra_options=[
+            '--legends', 'System 1,System 2,Sum,GMM,LLR,PLR-2',
+            '-S', '-e', '-la', '1e-2'
+        ])
+    elif name == 'roc-fusion-multi-modal-all':
+        return roc([
+            'scores/multi-modal/face/scores-dev',
+            'scores/multi-modal/face/scores-eval',
+            'scores/multi-modal/voice/scores-dev',
+            'scores/multi-modal/voice/scores-eval',
+            'scores/multi-modal/fingerprint/scores-dev',
+            'scores/multi-modal/fingerprint/scores-eval',
+            'scores/multi-modal/sum/standard/scores-dev',
+            'scores/multi-modal/sum/standard/scores-eval',
+            'scores/multi-modal/gmm/standard/scores-dev',
+            'scores/multi-modal/gmm/standard/scores-eval',
+            'scores/multi-modal/llr/standard/scores-dev',
+            'scores/multi-modal/llr/standard/scores-eval',
+            'scores/multi-modal/plr-2/standard/scores-dev',
+            'scores/multi-modal/plr-2/standard/scores-eval',
+        ],
+            'roc-fusion-multi-modoal-all',
+            extra_options=[
+            '--legends', 'Face,Voice,Fingerprint,Sum,GMM,LLR,PLR-2',
+            '-S', '-e', '-la', '1e-2'
+        ])
+
+    else:
+        raise ValueError("Unknown roc plot")
+
+
+def run_fusions():
+    # Sum Fusion with un-normalized scores
+    check_call(
+        'bin/bob fusion fuse -vvv -a sum.py:algorithm_no_normalize '
+        '--force -g dev -g eval -o scores/multi-algorithm/sum/no_normalize '
+        'scores/multi-algorithm/system-1-scores-dev '
+        'scores/multi-algorithm/system-1-scores-eval '
+        'scores/multi-algorithm/system-2-scores-dev '
+        'scores/multi-algorithm/system-2-scores-eval'.split())
+
+    # normalize scores
+    check_call(
+        'bin/bob fusion fuse -vvv -a preprocess.py:znorm '
+        '--force --skip-check -o scores/multi-algorithm/normalized/znrom/system-1/ '
+        'scores/multi-algorithm/system-1-scores-dev '
+        'scores/multi-algorithm/system-1-scores-dev '
+        'scores/multi-algorithm/system-1-scores-eval'.split())
+    check_call(
+        'bin/bob fusion fuse -vvv -a preprocess.py:znorm '
+        '--force --skip-check -o scores/multi-algorithm/normalized/znrom/system-2/ '
+        'scores/multi-algorithm/system-2-scores-dev '
+        'scores/multi-algorithm/system-2-scores-dev '
+        'scores/multi-algorithm/system-2-scores-eval'.split())
+
+    # Sum Fusion with normalized scores
+    check_call('bin/bob fusion fuse -vvv -a sum.py:algorithm_znorm '
+               '--force -o scores/multi-algorithm/sum/znorm '
+               'scores/multi-algorithm/system-1-scores-dev '
+               'scores/multi-algorithm/system-1-scores-dev '
+               'scores/multi-algorithm/system-1-scores-eval '
+               'scores/multi-algorithm/system-2-scores-dev '
+               'scores/multi-algorithm/system-2-scores-dev '
+               'scores/multi-algorithm/system-2-scores-eval'.split())
+
+    # different normalization approaches
+    for norm in ['znorm', 'tanh', 'standard']:
+        for sys in ['1', '2']:
+            check_call(
+                'bin/bob fusion fuse -vvv -a preprocess.py:{norm} '
+                '--force --skip-check -o scores/multi-algorithm/normalized/{norm}/system-{sys}/ '
+                'scores/multi-algorithm/system-{sys}-scores-dev '
+                'scores/multi-algorithm/system-{sys}-scores-dev '
+                'scores/multi-algorithm/system-{sys}-scores-eval'.format(
+                    sys=sys, norm=norm).split())
+        check_call('bin/bob fusion fuse -vvv -a sum.py:algorithm_{norm} '
+                   '--force -o scores/multi-algorithm/sum/{norm} '
+                   'scores/multi-algorithm/system-1-scores-dev '
+                   'scores/multi-algorithm/system-1-scores-dev '
+                   'scores/multi-algorithm/system-1-scores-eval '
+                   'scores/multi-algorithm/system-2-scores-dev '
+                   'scores/multi-algorithm/system-2-scores-dev '
+                   'scores/multi-algorithm/system-2-scores-eval'.format(
+                       norm=norm).split())
+
+    # run different fusion algorithms
+    for alg in ['gmm', 'llr', 'plr-2', 'plr-3']:
+        check_call('bin/bob fusion fuse -vvv -a {alg} '
+                   '--force -o scores/multi-algorithm/{alg}/{norm} '
+                   'scores/multi-algorithm/system-1-scores-dev '
+                   'scores/multi-algorithm/system-1-scores-dev '
+                   'scores/multi-algorithm/system-1-scores-eval '
+                   'scores/multi-algorithm/system-2-scores-dev '
+                   'scores/multi-algorithm/system-2-scores-dev '