Commit c81d9a9f authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Heatmap

parent 6f2b0c1c
Pipeline #58746 failed with stages
in 2 minutes and 24 seconds
......@@ -2,6 +2,8 @@
Some plotting demographic plotting mechanisms
"""
import seaborn as sns
from bob.bio.demographics.regularizers import demographic
from . import (
split_scores_by_variable,
compute_fmr_thresholds,
......@@ -322,6 +324,99 @@ def plot_fdr(
return fig
def plot_fmr_fnmr_heatmap(
fmrs,
fnmrs,
fdrs,
title,
heatmap_reference_fmr,
demographic_keys,
fmr_thresholds,
lookup_table=None,
):
"""
Plot the FMR and FNR heatmap
Parameters
----------
fmrs: dict
Dictionary containing the FMR values for each demographic comparison and each threshold
fnmrs: dict
Dictionary containing the FNMR values for each demographic comparison and each threshold
fdrs: list
List containing the FDR values
reference_threshold: float
Reference threshold used to compute FMR and FNRM
"""
fmr_heatmap = np.zeros((len(demographic_keys), len(demographic_keys)))
fnmr_heatmap = np.zeros((len(demographic_keys)))
for i, k_i in enumerate(demographic_keys):
key = f"{k_i}__{k_i}"
fnmr_heatmap[i] = fnmrs[key][heatmap_reference_fmr]
for j, k_j in enumerate(demographic_keys):
key = f"{k_i}__{k_j}"
fmr_heatmap[i, j] = fmrs[key][heatmap_reference_fmr]
fnmr_heatmap = np.diag(fnmr_heatmap)
## PLOT
(
fig,
(ax_fnmr, ax_fmr),
) = plt.subplots(1, 2, figsize=(8, 6))
fig.suptitle(
f"System: {title}; FMR and FNRM @ FMR={fmr_thresholds[heatmap_reference_fmr]}. FDR: {round(fdrs[heatmap_reference_fmr],3)}",
fontsize=14,
)
ticklabels = None
if lookup_table is not None:
demographic_names = dict(
set(
[
(i.split("__")[0], j.split("-")[0])
for i, j in list(lookup_table.items())
]
)
)
ticklabels = [demographic_names[k] for k in demographic_keys]
ax_fnmr.set_title("FNMR")
sns.heatmap(
fnmr_heatmap,
annot=True,
linewidths=0.5,
vmin=0,
vmax=0.5,
ax=ax_fnmr,
square=True,
xticklabels=ticklabels,
yticklabels=ticklabels,
cbar=False,
)
ax_fmr.set_title("FMR")
sns.heatmap(
fmr_heatmap,
annot=True,
linewidths=0.5,
vmin=0,
vmax=0.5,
ax=ax_fmr,
square=True,
xticklabels=ticklabels,
yticklabels=ticklabels,
)
return fig
def plot_fmr_fnmr_tradeoff(
negatives_dev,
positives_dev,
......@@ -377,7 +472,6 @@ def plot_fmr_fnmr_tradeoff(
"""
fmrs, fnmrs, fdrs = compute_fmr_fnmr_tradeoff(
negatives_dev,
positives_dev,
......
......@@ -2,8 +2,14 @@
Standar report API
"""
from bob.bio.demographics.regularizers import demographic
from .io import load
from .plot import plot_demographic_boxplot, plot_fmr_fnmr_tradeoff, plot_fdr
from .plot import (
plot_demographic_boxplot,
plot_fmr_fnmr_tradeoff,
plot_fdr,
plot_fmr_fnmr_heatmap,
)
from . import compute_fmr_thresholds
from matplotlib.backends.backend_pdf import PdfPages
......@@ -77,6 +83,7 @@ def standard_report(
percentile=0.01,
titles=None,
lookup_table=None,
heatmap_reference_fmr=2,
):
"""
Standard fairness report.
......@@ -119,6 +126,9 @@ def standard_report(
Look up table to be used in the demographic boxplot.
For instance, `W__W` -> White_White, `B__B` -> Black_Black, etc.
heatmap_reference_fmr: int
Reference value to be used in the heatmap (This value corresponds to an index on fmr_thresholds).
"""
......@@ -165,12 +175,23 @@ def standard_report(
title=title,
pre_computed_taus=taus,
)
pdf.savefig(fig)
fig = plot_fmr_fnmr_heatmap(
fmrs,
fnmrs,
fdrs,
title,
heatmap_reference_fmr,
demographic_keys=set([x.split("__")[0] for x in list(lookup_table.keys())]),
fmr_thresholds=fmr_thresholds,
lookup_table=lookup_table,
)
pdf.savefig(fig)
pretty_print(fmr_thresholds, fmrs, fnmrs, title, fdrs, lookup_table)
cache_fdrs.append(fdrs)
pdf.savefig(fig)
# Plotting the FDR
fig = plot_fdr(
negatives_eval,
......
......@@ -33,6 +33,7 @@ requirements:
- bob.measure
- bob.bio.base
- bob.bio.face
- seaborn
# place other runtime dependencies here (same as requirements.txt)
test:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment