Commit fe75d01e authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Optimized reports

parent 43ff622f
Pipeline #59085 failed with stages
in 19 minutes and 28 seconds
......@@ -25,6 +25,7 @@ def plot_demographic_boxplot(
label_lookup_table=None,
percentile=None,
title="",
pre_computed_taus=None,
):
"""
Plot the box-plots of the score distribution
......@@ -58,6 +59,8 @@ def plot_demographic_boxplot(
title="",
pre_compute_taus: list
Set of precomputed decision thresholds. If `None`, it will be computed on the fly
"""
......@@ -67,11 +70,14 @@ def plot_demographic_boxplot(
)
# Computing decision thresholds if we have any FMR
taus = (
compute_fmr_thresholds(negatives_dev, fmr_thresholds)
if fmr_thresholds is not None
else None
)
if pre_computed_taus is None:
taus = (
compute_fmr_thresholds(negatives_dev, fmr_thresholds)
if fmr_thresholds is not None
else None
)
else:
taus = pre_computed_taus
# Spliting the scores by cohort
negatives_dev_as_dict, positives_dev_as_dict = split_scores_by_variable(
......
......@@ -136,7 +136,7 @@ def standard_report(
# Computing and caching the decision thresholds
# This will be used in several plots
taus = [compute_fmr_thresholds(d, fmr_thresholds) for d in negatives_dev]
# taus = [compute_fmr_thresholds(d, fmr_thresholds) for d in negatives_dev]
cache_fdrs = []
......@@ -144,13 +144,26 @@ def standard_report(
assert len(titles) == len(negatives_dev)
assert len(titles) == len(negatives_eval)
for i, (n_dev, p_dev, n_eval, p_eval, t) in enumerate(
zip(negatives_dev, positives_dev, negatives_eval, positives_eval, taus)
taus = []
for i, (n_dev, p_dev, n_eval, p_eval) in enumerate(
zip(negatives_dev, positives_dev, negatives_eval, positives_eval)
):
# taus = [compute_fmr_thresholds(d, fmr_thresholds) for d in negatives_dev]
## Putting everything in the workers memory for future operations
# THis will speed up a lot the things
n_dev = n_dev.persist()
p_dev = p_dev.persist()
n_eval = n_eval.persist()
p_eval = p_eval.persist()
# Computing the decision thresholds
tau = compute_fmr_thresholds(n_dev, fmr_thresholds)
taus.append(tau)
title = None if titles is None else titles[i]
"""
fig = plot_demographic_boxplot(
negatives_dev=n_dev,
positives_dev=p_dev,
......@@ -161,9 +174,10 @@ def standard_report(
percentile=percentile,
fmr_thresholds=fmr_thresholds,
title=title,
pre_computed_taus=tau,
)
pdf.savefig(fig)
"""
#### PLOTTING THE FMR AND FNMR TRADE OFF
fig, fmrs, fnmrs, fdrs = plot_fmr_fnmr_tradeoff(
......@@ -175,7 +189,7 @@ def standard_report(
positives_eval=p_eval,
label_lookup_table=lookup_table,
title=title,
pre_computed_taus=t,
pre_computed_taus=tau,
)
pdf.savefig(fig)
......@@ -287,6 +301,9 @@ def morph_report(
negatives_dev = [filter_out(n) for n in negatives_dev]
positives_dev = [filter_out(n) for n in positives_dev]
negatives_eval = [n if n is None else filter_out(n) for n in negatives_eval]
positives_eval = [n if n is None else filter_out(n) for n in positives_eval]
standard_report(
negatives_dev,
positives_dev,
......
......@@ -40,11 +40,12 @@ from bob.bio.face.embeddings.mxnet import arcface_insightFace_lresnet100
### Defining baselines
score_level_baselines = dict()
# score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
# score_level_baselines["facenet-sanderberg"] = facenet_sanderberg_20170512_110547
# score_level_baselines[
# "resnet50-msceleb-arcface-20210521"
# ] = resnet50_msceleb_arcface_20210521
# score_level_baselines["iresnet50"] = iresnet50
score_level_baselines["facenet-sanderberg"] = facenet_sanderberg_20170512_110547
score_level_baselines[
"resnet50-msceleb-arcface-20210521"
] = resnet50_msceleb_arcface_20210521
score_level_baselines["iresnet100"] = iresnet100
score_level_baselines["iresnet50"] = iresnet50
score_level_baselines["attentionnet"] = AttentionNet
score_level_baselines["resnest"] = ResNeSt
......@@ -96,7 +97,7 @@ def execute(
### Running the baseline
experiment_path = os.path.join(output_path, "score-norm", database.name, b)
experiment_path = os.path.join(output_path, "score-norm", database.protocol, b)
execute_vanilla_biometrics_score_normalization(
score_level_baselines[b](
......@@ -117,24 +118,24 @@ def execute(
score_normalization_type="znorm",
)
execute_vanilla_biometrics_score_normalization(
score_level_baselines[b](
annotation_type=database.annotation_type,
fixed_positions=database.fixed_positions,
),
database,
dask_client,
groups=groups,
output=experiment_path,
write_metadata_scores=True,
checkpoint=True,
dask_partition_size=100,
dask_n_workers=20,
checkpoint_dir=None,
top_norm=top_norm,
top_norm_score_fraction=top_norm_score_fraction,
score_normalization_type="tnorm",
)
# execute_vanilla_biometrics_score_normalization(
# score_level_baselines[b](
# annotation_type=database.annotation_type,
# fixed_positions=database.fixed_positions,
# ),
# database,
# dask_client,
# groups=groups,
# output=experiment_path,
# write_metadata_scores=True,
# checkpoint=True,
# dask_partition_size=100,
# dask_n_workers=20,
# checkpoint_dir=None,
# top_norm=top_norm,
# top_norm_score_fraction=top_norm_score_fraction,
# score_normalization_type="tnorm",
# )
if dask_client is not None:
dask_client.shutdown()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment