Commit d2626448 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Optimizing the reporting section

parent 963f9456
......@@ -3,5 +3,5 @@ from .scores import (
split_scores_by_variable,
fairness_discrepancy_rate,
compute_fdr,
compute_fmr_fnmr_tradeoff,
)
......@@ -2,9 +2,7 @@ from bob.bio.face.pytorch.head import ArcFace
### train_dataloader AND BACKBONES needs to be loaded before
#NUM_CLASS = train_dataloader.dataset.n_classes
#WEIGHT = train_dataloader.dataset.get_demographic_class_weights()
NUM_CLASS = len(list(train_dataloader.dataset.demographic_keys.values()))
NUM_CLASS = len(list(train_dataset.demographic_keys.values()))
identity_head = ArcFace(feat_dim=backbone.features.num_features, num_class=NUM_CLASS)
......
......@@ -4,7 +4,7 @@ from bob.bio.face.pytorch.head import MagFace
### train_dataloader AND BACKBONES needs to be loaded before
#NUM_CLASS = train_dataloader.dataset.n_classes
NUM_CLASS = len(list(train_dataloader.dataset.demographic_keys.values()))
NUM_CLASS = len(list(train_dataset.demographic_keys.values()))
#WEIGHT = train_dataloader.dataset.get_demographic_class_weights()
identity_head = MagFace(feat_dim=backbone.features.num_features, num_class=NUM_CLASS)
......
......@@ -2,8 +2,8 @@ from bob.bio.face.pytorch.head import ArcFace
### train_dataloader AND BACKBONES needs to be loaded before
NUM_CLASS = train_dataloader.dataset.n_classes
WEIGHT = train_dataloader.dataset.get_demographic_class_weights()
NUM_CLASS = train_dataset.n_classes
WEIGHT = train_dataset.get_demographic_class_weights()
identity_head = ArcFace(feat_dim=backbone.features.num_features, num_class=NUM_CLASS)
......
......@@ -2,9 +2,8 @@ from bob.bio.face.pytorch.head import MagFace
### train_dataloader AND BACKBONES needs to be loaded before
NUM_CLASS = train_dataloader.dataset.n_classes
WEIGHT = train_dataloader.dataset.get_demographic_class_weights()
NUM_CLASS = train_dataset.n_classes
WEIGHT = train_dataset.get_demographic_class_weights()
identity_head = MagFace(feat_dim=backbone.features.num_features, num_class=NUM_CLASS)
......
......@@ -6,20 +6,18 @@ BASE_PATH=/idiap/temp/tpereira/2.FRDemographics/regularization/models/contrastiv
BACKBONE=iresnet50
#jman submit --name C.IRESNET50 -q gpu --sge-extra-args="-l hostname=vgni001|vgni00[1-9]|vgni01[0-9]|vgni02[0-9]|vgni03[0-2]" --
#./bin/bob bio demographics train-regularization-level-fairness train-contrastive \
# $BASE_PATH/$BACKBONE \
# --database ./bob/bio/demographics/config/train/databases/mobio/mobio-male-female.py \
# --identity-backbone ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
# --training-config ./bob/bio/demographics/config/train/training_configs/regular_sgd_crossentropy.py \
# --max-epochs 50 --batch-size 64 \
# --max-positive-pairs-per-subject 20 --negative-pairs-per-subject 20
jman submit --name C.IRESNET50 -q gpu --sge-extra-args="-l hostname=vgni001|vgni00[1-9]|vgni01[0-9]|vgni02[0-9]|vgni03[0-2]|vgnh00[1-8]|vgng00[1-8]|vgnf00[1-9]|vgnf01[1-6]" -- ./bin/bob bio demographics train-regularization-level-fairness train-contrastive \
$BASE_PATH/$BACKBONE \
--database ./bob/bio/demographics/config/train/databases/mobio/mobio-male-female.py \
--identity-backbone ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
--training-config ./bob/bio/demographics/config/train/training_configs/regular_sgd_crossentropy.py \
--max-epochs 50 --batch-size 64 \
--max-positive-pairs-per-subject 20 --negative-pairs-per-subject 20
BACKBONE=iresnet100
jman submit --name C.IRESNET100 -q gpu --sge-extra-args="-l hostname=vgni001|vgni00[1-9]|vgni01[0-9]|vgni02[0-9]|vgni03[0-2]" -- ./bin/bob bio demographics train-regularization-level-fairness train-contrastive \
jman submit --name C.IRESNET100 -q gpu --sge-extra-args="-l hostname=vgni001|vgni00[1-9]|vgni01[0-9]|vgni02[0-9]|vgni03[0-2]|vgnh00[1-8]|vgng00[1-8]|vgnf00[1-9]|vgnf01[1-6]" -- ./bin/bob bio demographics train-regularization-level-fairness train-contrastive \
$BASE_PATH/$BACKBONE \
--database ./bob/bio/demographics/config/train/databases/mobio/mobio-male-female.py \
--identity-backbone ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
......
......@@ -3,9 +3,7 @@ BASE_PATH=/idiap/temp/tpereira/2.FRDemographics/regularization/models/demographi
BACKBONE=iresnet34
#jman submit --name D.IRESNET34 -q gpu --
./bin/bob bio demographics train-regularization-level-fairness train-demographics $BASE_PATH/$BACKBONE \
jman submit --name VGG-D.IRESNET34 -q gpu -- ./bin/bob bio demographics train-regularization-level-fairness train-demographics $BASE_PATH/$BACKBONE \
-d ./bob/bio/demographics/config/train/databases/vgg2/vgg2-short-with-validation.py \
-b ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
-h ./bob/bio/demographics/config/train/demographic_heads/arcface.py \
......@@ -16,13 +14,11 @@ BACKBONE=iresnet34
BACKBONE=iresnet50
#jman submit --name D.IRESNET50 -q gpu --
#./bin/bob bio demographics train-regularization-level-fairness train-demographics $BASE_PATH/$BACKBONE \
# -d ./bob/bio/demographics/config/train/databases/vgg2/vgg2-short-with-validation.py \
# -b ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
# -h ./bob/bio/demographics/config/train/demographic_heads/arcface.py \
# -t ./bob/bio/demographics/config/train/training_configs/regular_sgd_crossentropy.py
jman submit --name VGG-D.IRESNET50 -q gpu -- ./bin/bob bio demographics train-regularization-level-fairness train-demographics $BASE_PATH/$BACKBONE \
-d ./bob/bio/demographics/config/train/databases/vgg2/vgg2-short-with-validation.py \
-b ./bob/bio/demographics/config/train/backbones/$BACKBONE.py \
-h ./bob/bio/demographics/config/train/demographic_heads/arcface.py \
-t ./bob/bio/demographics/config/train/training_configs/regular_sgd_crossentropy.py
......
......@@ -2,9 +2,15 @@
Some plotting demographic plotting mechanisms
"""
from . import split_scores_by_variable, compute_fmr_thresholds, compute_fdr
from . import (
split_scores_by_variable,
compute_fmr_thresholds,
compute_fdr,
compute_fmr_fnmr_tradeoff,
)
import matplotlib.pyplot as plt
import bob.measure
import numpy as np
def plot_demographic_boxplot(
......@@ -164,9 +170,9 @@ def plot_demographic_boxplot(
title = (
title
if percentile is None
else title + f" - boxplot at percentile = {percentile}"
else f"System: {title}. Box plot scores at percentile = {percentile}"
)
# fig.suptitle(title)
fig.suptitle(title)
negatives_dev_as_dict
if has_eval():
......@@ -225,6 +231,7 @@ def plot_fdr(
beta=0,
taus=None,
title=None,
cached_fdr=None,
):
"""
Return the Fairness Discrepancy Rate PLOT from the paper
......@@ -265,6 +272,9 @@ def plot_fdr(
Decision thresholds. If `None`, it will be computed on the fly
using `fmr_thresholds`
cached_fdr: list of lists
If not `None`, it will be used instead of computing the FDR
"""
from . import fairness_discrepancy_rate
......@@ -272,26 +282,28 @@ def plot_fdr(
assert len(positives) == len(negatives) == len(labels)
title = "Fairness Discrepancy Rate" if title is None else title
if taus is None:
taus = [compute_fmr_thresholds(n, fmr_thresholds) for n in negatives]
fdrs = [
fairness_discrepancy_rate(
neg,
pos,
variable_suffix,
fmr_thresholds,
fdr_fn=fdr_fn,
alpha=alpha,
beta=beta,
taus=t,
)
for neg, pos, t in zip(negatives, positives, taus)
]
fdrs = cached_fdr
if cached_fdr is None:
if taus is None:
taus = [compute_fmr_thresholds(n, fmr_thresholds) for n in negatives]
fdrs = [
fairness_discrepancy_rate(
neg,
pos,
variable_suffix,
fmr_thresholds,
fdr_fn=fdr_fn,
alpha=alpha,
beta=beta,
taus=t,
)
for neg, pos, t in zip(negatives, positives, taus)
]
# fig, ax = plt.subplots(figsize=(16, 8))
fig, ax = plt.subplots(figsize=(8, 6))
# fig.suptitle(title)
[
plt.semilogx(fmr_thresholds, f, label=l, linewidth=2)
......@@ -299,42 +311,14 @@ def plot_fdr(
]
[plt.scatter(fmr_thresholds, f) for f in fdrs]
print(fdrs)
plt.ylabel("$FDR(\\tau)$", fontsize=18)
plt.xlabel("$\\tau=FMR_{10^{-x}}$", fontsize=18)
plt.grid(True)
fig.suptitle(title)
plt.legend()
### PARETO PLOT
"""
FMRS = []
FNMRS = []
for neg, pos, tau in zip(negatives, positives, taus):
fmrs = []
fnmrs = []
for t in tau:
fmr, fnmr = bob.measure.farfrr(
neg["score"].compute().to_numpy(), pos["score"].compute().to_numpy(), t
)
fmrs.append(fmr)
fnmrs.append(fnmr)
FMRS.append(fmrs)
FNMRS.append(fnmrs)
ax = plt.axes(projection="3d")
for fmr, fnmr, fdr in zip(FMRS, FNMRS, fdrs):
ax.plot3D(fmr, fnmr, fdr)
ax.scatter3D(fmr, fnmr, fdr)
ax.set_xlabel("$FMR$")
ax.set_ylabel("$FNMR$")
ax.set_zlabel("$FDR$")
"""
return fig
......@@ -347,59 +331,76 @@ def plot_fmr_fnmr_tradeoff(
positives_eval=None,
label_lookup_table=None,
title="False Match and False non Match trade-off per demographic",
print_fmr_fnmr=False,
fdr_fn=compute_fdr,
pre_computed_taus=None,
):
def has_eval():
return negatives_eval is not None and positives_eval is not None
"""
Computes and plot the FMR and FNMR trade-off
# Computing decision thresholds if we have any FMR
taus = compute_fmr_thresholds(negatives_dev, fmr_thresholds)
Parameters
----------
negatives_dev: dataframe
Pandas Dataframe containing the negative scores (or impostor scores, or even non-mated scores)
# If we have an evaluation set, do the plots with the evaluation set,
# otherwise uses the development set
if has_eval():
negatives_as_dict, positives_as_dict = split_scores_by_variable(
negatives_eval, positives_eval, variable_suffix
)
else:
negatives_as_dict, positives_as_dict = split_scores_by_variable(
negatives_dev, positives_dev, variable_suffix
)
positives_dev: dataframe
Pandas Dataframe containing the positive scores (or genuines scores, or even mated scores)
# Iterating ONLY on comparisons of the same
# demographic group
fmrs = dict()
fnmrs = dict()
# for key in positives_as_dict:
for key in negatives_as_dict:
fmrs[key] = []
if key in positives_as_dict:
fnmrs[key] = []
for t in taus:
if key in positives_as_dict:
fmr, fnmr = bob.measure.farfrr(
negatives_as_dict[key]["score"].compute().to_numpy(),
positives_as_dict[key]["score"].compute().to_numpy(),
t,
)
fnmrs[key].append(fnmr)
else:
fmr, _ = bob.measure.farfrr(
negatives_as_dict[key]["score"].compute().to_numpy(),
[0.0],
t,
)
fmrs[key].append(fmr)
variable_suffix: str
Variable used to split the data
fmr_thresholds: list
List containing the FMR operational points
negatives_eval: dataframe
Pandas Dataframe containing the negative scores from evaluation set (or impostor scores, or even non-mated scores)
positives_eval: dataframe
Pandas Dataframe containing the positive scores from evaluation set (or genuines scores, or even mated scores)
fdr_fn:
Function used to compute the FDR
pre_computed_taus: list
List of decision thresholds. If `None`, it will be computed on the fly
Returns
-------
fig: matplotlib figure
The figure containing the FMR and FNMR trade-off
fmrs: dict
Dictionary containing the FMRs for each demographic
fnms: dict
Dictionary containing the FNMRs for each demographic
fdrs: list
List of FDRs for each system
"""
fmrs, fnmrs, fdrs = compute_fmr_fnmr_tradeoff(
negatives_dev,
positives_dev,
variable_suffix,
fmr_thresholds,
negatives_eval,
positives_eval,
fdr_fn,
pre_computed_taus,
)
# Plotting the FMR and FNMR in two
# separated subplots
fig, ax = plt.subplots(figsize=(16, 8))
set_name = "dev" if negatives_eval is None else "eval"
title = f"System: {title}. FMR and FNMR trade-off on {set_name} set"
fig.suptitle(title)
# LABELS FOR FNMR
labels_fnmr = list(positives_as_dict.keys())
labels_fnmr = list(fnmrs.keys())
if label_lookup_table is not None:
labels_fnmr = [label_lookup_table[l] for l in labels_fnmr]
......@@ -414,7 +415,7 @@ def plot_fmr_fnmr_tradeoff(
plt.grid(True)
# LABELS FOR FMR
labels_fmr = list(negatives_as_dict.keys())
labels_fmr = list(fmrs.keys())
if label_lookup_table is not None:
labels_fmr = [label_lookup_table[l] for l in labels_fmr]
......@@ -426,22 +427,4 @@ def plot_fmr_fnmr_tradeoff(
plt.grid(True)
plt.legend()
## Printing
if print_fmr_fnmr:
def print_table(header, fmrs, title, labels):
from tabulate import tabulate
print(title)
content = [
[l] + [round(value, 3) for value in fmrs[key]]
for key, l in zip(fmrs, labels)
]
print(tabulate([header] + content, tablefmt="rst"))
header = ["Ethnicities"] + fmr_thresholds
print_table(header, fmrs, title=f"{title} - FMR", labels=labels_fmr)
print_table(header, fnmrs, title=f"{title} - FNMR", labels=labels_fnmr)
return fig
return fig, fmrs, fnmrs, fdrs
......@@ -39,7 +39,7 @@ class ContrastiveModel(pl.LightningModule):
backbone_checkpoint_file=None,
weight_contrastive_loss=1.0,
weight_calibration_loss=1.0,
demographic_weights=0,
demographic_weights=None,
**kwargs,
):
super(ContrastiveModel, self).__init__(
......@@ -175,6 +175,10 @@ class ContrastiveModel(pl.LightningModule):
demographic_loss = self.calibration_loss(
after_calibration_scores[:, 0], after_calibration_labels
)
if self.demographic_weights is not None:
demographic_loss *= self.demographic_weights[i]
# accumulated_loss += demographic_loss
calibration_loss.append(demographic_loss)
self.log(f"train/demographic_loss_{i}", demographic_loss)
......
......@@ -21,7 +21,7 @@ class DemographicModel(BackboneHeadModel):
identity_head=None,
loss_fn=None,
optimizer_fn=None,
backbone_checkpoint_path=None,
backbone_checkpoint_file=None,
**kwargs,
):
super(DemographicModel, self).__init__(
......@@ -31,7 +31,7 @@ class DemographicModel(BackboneHeadModel):
optimizer_fn=optimizer_fn,
**kwargs,
)
self.backbone_checkpoint_path = backbone_checkpoint_path
self.backbone_checkpoint_file = backbone_checkpoint_file
def training_step(self, batch, batch_idx):
......
......@@ -7,6 +7,9 @@ from .plot import plot_demographic_boxplot, plot_fmr_fnmr_tradeoff, plot_fdr
from . import compute_fmr_thresholds
from matplotlib.backends.backend_pdf import PdfPages
import logging
logger = logging.getLogger(__name__)
def load_dev_eval_scores(scores_dev, scores_eval, load_fn=load):
......@@ -35,52 +38,101 @@ def load_dev_eval_scores(scores_dev, scores_eval, load_fn=load):
return negatives_dev, positives_dev, negatives_eval, positives_eval
def meds_report(
scores_dev,
def pretty_print(fmr_thresholds, fmrs, fnmrs, title, fdrs, label_lookup_table):
from tabulate import tabulate
def print_table(header, fmrs, title, labels):
print(title)
content = [
[l] + [round(value, 3) for value in fmrs[key]]
for key, l in zip(fmrs, labels)
]
print(tabulate([header] + content, tablefmt="rst"))
labels_fmr = [
k if label_lookup_table is None else label_lookup_table[k]
for k in list(fmrs.keys())
]
labels_fnmr = [
k if label_lookup_table is None else label_lookup_table[k]
for k in list(fnmrs.keys())
]
header = ["Ethnicities"] + fmr_thresholds
print_table(header, fmrs, title=f"{title} - FMR", labels=labels_fmr)
print_table(header, fnmrs, title=f"{title} - FNMR", labels=labels_fnmr)
print(tabulate([["FDR"] + [(str(x)) for x in fdrs]]))
def standard_report(
negatives_dev,
positives_dev,
output_filename,
scores_eval=None,
variable_suffix,
negatives_eval=None,
positives_eval=None,
fmr_thresholds=[10 ** i for i in list(range(-6, 0))],
percentile=0.01,
titles=None,
lookup_table=None,
):
"""
Standard fairness report.
It will print:
1. The FDR plot
2. The demographic boxplot
3. The FMR and FNMR trade off
label_lookup_table = {
"W__W": "White-White",
"W__B": "White-Black",
"B__W": "Black-White",
"B__B": "Black-Black",
}
Parameters:
-----------
negatives_dev: list of pandas dataframes
Pandas dataframes containing the negative samples
variable_suffix = "rac"
positives_dev: list of pandas dataframes
Pandas dataframes containing the positive samples
pdf = PdfPages(output_filename)
output_filename: str
Path to the output file
negatives_dev, positives_dev, negatives_eval, positives_eval = load_dev_eval_scores(
scores_dev, scores_eval
)
variable_suffix: str
Variable suffix to be used to split the score file.
E.g. rac, sex, gender, etc.
if negatives_eval[0] is None:
negatives_dev: list of pandas dataframes
Pandas dataframes containing the negative samples (evaluation set. Default to `None`)
# Compute FDR on the same set if there's no evaluation set
fig = plot_fdr(
negatives_dev, positives_dev, titles, variable_suffix, fmr_thresholds
)
else:
# If there is evaluation set
# compute the decision thresholds
positives_dev: list of pandas dataframes
Pandas dataframes containing the positive samples (evaluation set. Default to `None`)
taus = [compute_fmr_thresholds(d, fmr_thresholds) for d in negatives_dev]
fmr_thresholds: list of float
List of FMR thresholds to be used in the plot
fig = plot_fdr(
negatives_eval,
positives_eval,
titles,
variable_suffix,
fmr_thresholds,
taus=taus,
)
percentile: float
Percentile to be used in the demographic boxplot
pdf.savefig(fig)
titles: list of str
List of titles for the plots
lookup_table: dict
Look up table to be used in the demographic boxplot.
For instance, `W__W` -> White_White, `B__B` -> Black_Black, etc.
"""
pdf = PdfPages(output_filename)
# Computing and caching the decision thresholds
# This will be used in several plots
taus = [compute_fmr_thresholds(d, fmr_thresholds) for d in negatives_dev]
cache_fdrs = []
if titles is not None:
assert len(titles) == len(negatives_dev)
assert len(titles) == len(negatives_eval)
for i, (n_dev, p_dev, n_eval, p_eval) in enumerate(
zip(negatives_dev, positives_dev, negatives_eval, positives_eval)
......@@ -94,7 +146,7 @@ def meds_report(
variable_suffix=variable_suffix,
negatives_eval=n_eval,
positives_eval=p_eval,
label_lookup_table=label_lookup_table,
label_lookup_table=lookup_table,
percentile=percentile,
fmr_thresholds=fmr_thresholds,
title=title,
......@@ -102,22 +154,74 @@ def meds_report(
pdf.savefig(fig)
#### PLOTTING THE FMR AND FNMR TRADE OFF
fig = plot_fmr_fnmr_tradeoff(
fig, fmrs, fnmrs, fdrs = plot_fmr_fnmr_tradeoff(
n_dev,
p_dev,
variable_suffix=variable_suffix,
fmr_thresholds=fmr_thresholds,
negatives_eval=n_eval,
positives_eval=p_eval,
label_lookup_table=label_lookup_table,
print_fmr_fnmr=True,
label_lookup_table=lookup_table,
title=title,
pre_computed_taus=taus,
)
pretty_print(fmr_thresholds, fmrs, fnmrs, title, fdrs, lookup_table)
cache_fdrs.append(fdrs)
pdf.savefig(fig)