[Doc][Multipie] Adding leaderboard results, instructions for running the...

[Doc][Multipie] Adding leaderboard results, instructions for running the baselines, and example script for pose analysis

doc/leaderboard/multipie.rst

@@ -40,7 +40,7 @@ of 18 different possible flashes.
 
 The following picture showcase the positioning of the cameras (in yellow) and of the flashes (in white).
 
-.. image:: img/multipie_setup.jpg
+.. image:: img/multipie/multipie_setup.jpg
     :width: 620px
     :align: center
     :height: 200px
@@ -117,6 +117,116 @@ Benchmarks
 Run the baselines
 *****************
 
+You can run the Multipie baselines command with a simple command such as:
+
+.. code-block:: bash
+
+    bob bio pipeline vanilla-biometrics multipie gabor_graph -m -l sge
+
+Note that the default protocol implemented in the resource is the U protocol.
+The pose protocol is also available using
+
+.. code-block:: bash
+
+    bob bio pipeline vanilla-biometrics multipie_pose gabor_graph -m -l sge
+
+For the other protocols, one has to define its own configuration file (e.g.: *multipie_M.py*) as follows:
+
+.. code-block:: python
+
+    from bob.bio.face.database import MultipieDatabase
+    database = MultipieDatabase(protocol="M")
+
+then point to it when calling the pipeline execution:
+
+.. code-block:: bash
+
+    bob bio pipeline vanilla-biometrics multipie_M.py gabor_graph -m -l sge
+
+
+
 Leaderboard
 ***********
 
+Protocol M
+----------
+
+.. csv-table:: Protocol M
+   :file: table/multipie/multipie_M.csv
+   :header-rows: 1
+
+.. image:: img/multipie/multipie_M_DET_dev.png
+    :align: center
+    :alt: Multipie M - DET dev
+
+.. image:: img/multipie/multipie_M_DET_eval.png
+    :align: center
+    :alt: Multipie M - DET eval
+
+Protocol U
+----------
+
+.. csv-table:: Protocol U
+   :file: table/multipie/multipie_U.csv
+   :header-rows: 1
+
+.. image:: img/multipie/multipie_U_DET_dev.png
+    :align: center
+    :alt: Multipie U - DET dev
+
+.. image:: img/multipie/multipie_U_DET_eval.png
+    :align: center
+    :alt: Multipie U - DET eval
+
+Protocol E
+----------
+
+.. csv-table:: Protocol E
+   :file: table/multipie/multipie_E.csv
+   :header-rows: 1
+
+.. image:: img/multipie/multipie_E_DET_dev.png
+    :align: center
+    :alt: Multipie E - DET dev
+
+.. image:: img/multipie/multipie_E_DET_eval.png
+    :align: center
+    :alt: Multipie E - DET eval
+
+Protocol P
+----------
+
+.. csv-table:: Protocol P
+   :file: table/multipie/multipie_P.csv
+   :header-rows: 1
+
+.. image:: img/multipie/multipie_P_DET_dev.png
+    :align: center
+    :alt: Multipie P - DET dev
+
+.. image:: img/multipie/multipie_P_DET_eval.png
+    :align: center
+    :alt: Multipie P - DET eval
+
+For the pose protocol specifically, we can perform a more detailed study
+to assess angle-wise performance of the various FR systems.
+Hereafter is an example code to run this type of analysis, as well 
+as the results. This code is also available as a Jupytext-compatible
+.py file under `./script/multipie/pose_analysis.py`, that can be loaded 
+as a Jupyter notebook.
+
+.. raw:: html
+
+   <details>
+   <summary> Uncollapse : Multipie pose analysis </summary>
+   
+.. raw:: html
+    :file: script/multipie/pose_analysis.html
+
+.. raw:: html
+
+   </details>
+
+
+
+

doc/leaderboard/script/multipie/pose_analysis.py

+# ---
+# jupyter:
+#   jupytext:
+#     formats: ipynb,py:light
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.7.1
+#   kernelspec:
+#     display_name: bob_pipelines
+#     language: python
+#     name: bob_pipelines
+# ---
+
+import os
+import pandas as pd
+import bob.measure
+import numpy as np
+import matplotlib as mpl
+mpl.rcParams.update({'font.size': 14})
+import matplotlib.pyplot as plt
+# %matplotlib inline
+
+# ### Select baselines
+
+baselines = {
+    'Gabor graph': 'gabor_graph',
+    'LGBPHS': 'lgbphs',
+    'FaceNet (D. Sandberg)': 'facenet-sanderberg',
+    'Casia-Webface - Inception Resnet v1': 'inception-resnetv1-casiawebface',
+    'Casia-Webface - Inception Resnet v2': 'inception-resnetv2-casiawebface',
+    'MSCeleb - Inception Resnet v1': 'inception-resnetv1-msceleb',
+    'MSCeleb - Inception Resnet v2': 'inception-resnetv2-msceleb'
+}
+
+# ### Define some utilitary functions
+
+# +
+results_dir = './results/'
+
+# Function to load the scores in CSV format
+def load_scores(baseline, protocol, group):
+    scores = pd.read_csv(os.path.join(results_dir, 
+                                      'multipie_{}'.format(protocol), 
+                                      baseline, 
+                                      'scores-{}.csv'.format(group)))
+    return scores
+
+# Function to separate genuines from impostors
+def split(df):
+    impostors = df[df['probe_reference_id'] != df['bio_ref_reference_id']]
+    genuines = df[df['probe_reference_id'] == df['bio_ref_reference_id']]
+    return impostors, genuines
+
+
+# -
+
+# ### Establish Camera to Angle conversion
+
+cameras = ['11_0', '12_0','09_0','08_0','13_0','14_0','05_1','05_0', '04_1','19_0','20_0','01_0','24_0']
+angles = np.linspace(-90, 90, len(cameras))
+camera_to_angle = dict(zip(cameras, angles))
+
+# ### Run pose analysis and plot results
+
+# +
+# Figure for Dev plot
+fig1 = plt.figure(figsize=(8, 6))
+# Figure for Eval plot
+fig2 = plt.figure(figsize=(8, 6))
+for name, baseline in baselines.items():
+    
+    # Load the score files and fill in the angle associated to each camera
+    dev_scores = load_scores(baseline, 'P', 'dev')
+    eval_scores = load_scores(baseline, 'P', 'eval')
+    dev_scores['angle'] = dev_scores['probe_camera'].map(camera_to_angle)
+    eval_scores['angle'] = eval_scores['probe_camera'].map(camera_to_angle)
+    
+    angles = []
+    dev_hters = []
+    eval_hters = []
+    # Run the analysis per view angle
+    for (angle, dev_df), (_, eval_df) in zip(dev_scores.groupby('angle'), eval_scores.groupby('angle')):
+        # Separate impostors from genuines
+        dev_impostors, dev_genuines = split(dev_df)
+        eval_impostors, eval_genuines = split(eval_df)
+        
+        # Compute the min. HTER threshold on the Dev set
+        threshold = bob.measure.min_hter_threshold(dev_impostors['score'], dev_genuines['score'])
+        
+        # Compute the HTER for the Dev and Eval set at this particular threshold
+        dev_far, dev_frr = bob.measure.farfrr(dev_impostors['score'], dev_genuines['score'], threshold)
+        eval_far, eval_frr = bob.measure.farfrr(eval_impostors['score'], eval_genuines['score'], threshold)
+        angles.append(angle)
+        dev_hters.append(1/2 * (dev_far + dev_frr))
+        eval_hters.append(1/2 * (eval_far + eval_frr))
+    
+    # Update plots
+    plt.figure(1)   
+    plt.plot(angles, dev_hters, label=name, marker='x')
+    
+    plt.figure(2)
+    plt.plot(angles, eval_hters, label=name, marker='x')
+
+# Plot finalization
+plt.figure(1)
+plt.title('Dev. min. HTER')
+plt.xlabel('Angle')
+plt.ylabel('Min HTER')
+plt.legend()
+plt.grid()
+
+plt.figure(2)
+plt.title('Eval. HTER @Dev min. HTER')
+plt.xlabel('Angle')
+plt.ylabel('HTER')
+plt.legend()
+plt.grid()

doc/leaderboard/table/multipie/multipie_E.csv

+Baseline,EER (dev),HTER (eval)
+Gabor graph,14.24%,15.56%
+LGBPHS,13.36%,13.78%
+FaceNet (D. Sandberg),5.21%,6.64%
+Casia-Webface - Inception Resnet v1,12.89%,13.67%
+Casia-Webface - Inception Resnet v2,2.11%,3.07%
+MSCeleb - Inception Resnet v1,6.27%,6.60%
+MSCeleb - Inception Resnet v2,3.82%,4.46%

doc/leaderboard/table/multipie/multipie_M.csv

+Baseline,EER (dev),HTER (eval)
+Gabor graph,1.26%,3.68%
+LGBPHS,1.17%,3.06%
+FaceNet (D. Sandberg),1.56%,2.74%
+Casia-Webface - Inception Resnet v1,3.83%,5.98%
+Casia-Webface - Inception Resnet v2,0.76%,0.74%
+MSCeleb - Inception Resnet v1,1.56%,2.19%
+MSCeleb - Inception Resnet v2,3.09%,7.09%

doc/leaderboard/table/multipie/multipie_P.csv

+Baseline,EER (dev),HTER (eval)
+Gabor graph,44.83%,45.66%
+LGBPHS,48.23%,48.36%
+FaceNet (D. Sandberg),14.46%,13.93%
+Casia-Webface - Inception Resnet v1,37.23%,37.11%
+Casia-Webface - Inception Resnet v2,13.04%,13.12%
+MSCeleb - Inception Resnet v1,13.58%,14.76%
+MSCeleb - Inception Resnet v2,5.17%,5.22%

doc/leaderboard/table/multipie/multipie_U.csv

+Baseline,EER (dev),HTER (eval)
+Gabor graph,6.25%,7.44%
+LGBPHS,6.02%,6.85%
+FaceNet (D. Sandberg),3.31%,3.56%
+Casia-Webface - Inception Resnet v1,9.50%,9.43%
+Casia-Webface - Inception Resnet v2,0.80%,1.46%
+MSCeleb - Inception Resnet v1,3.97%,3.10%
+MSCeleb - Inception Resnet v2,3.82%,6.54%