Commit d68c779b authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Implementing different score calibration mechanisms

parent c478e2dd
"""The main entry for bob.bio (click-based) scripts.
"""
import click
import pkg_resources
from click_plugins import with_plugins
from bob.extension.scripts.click_helper import AliasedGroup
@with_plugins(pkg_resources.iter_entry_points("bob.bio.demographics.calibration.cli"))
@click.group(cls=AliasedGroup)
def calibration():
"""Calibration by category commands"""
pass
import click
from bob.bio.base.pipelines.vanilla_biometrics import (
CategoricalCalibration,
WeibullCalibration,
LLRCalibration,
)
def calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
field_name,
field_values,
):
if calibrator == "linear":
calibrator = LLRCalibration
elif calibrator == "weibull":
calibrator = WeibullCalibration
else:
raise ValueError(f"Invalid calibrator {calibrator}")
if quantile_factor=="none":
quantile_factor = None
else:
quantile_factor = float(quantile_factor)
calibrator = CategoricalCalibration(
field_name=field_name,
field_values=field_values,
quantile_factor=quantile_factor,
fit_estimator=calibrator,
)
calibrator = calibrator.fit(fit_score_file)
calibrator.transform(transform_score_files, output_score_files)
@click.command()
@click.argument("fit-score-file")
@click.option(
"--transform-score-files",
required=True,
multiple=True,
help="List of scores to be calibrated",
)
@click.option(
"--output-score-files",
required=True,
multiple=True,
help="Output scores.",
)
@click.option(
"--calibrator",
type=click.Choice(["linear", "weibull"], case_sensitive=False),
default="linear",
help="Calibrators.",
)
@click.option(
"--quantile-factor",
default="1.5",
help="Quantile factor used to pic the scores that fits the calibrator",
)
def mobio(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
):
"""
Calibrates scores coming from experiments using MOBIO database.
"""
#field_values = ["Asian", "African", "Indian", "Caucasian"]
#field_name = "race"
field_values = ["m", "f"]
field_name = "gender"
calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
field_name,
field_values,
)
"""The main entry for bob.bio (click-based) scripts.
"""
import click
import pkg_resources
from click_plugins import with_plugins
from bob.extension.scripts.click_helper import AliasedGroup
@with_plugins(pkg_resources.iter_entry_points("bob.bio.demographics.reports.cli"))
@click.group(cls=AliasedGroup)
def reports():
"""Reports for some databases"""
pass
"""The main entry for bob.bio (click-based) scripts.
"""
import click
import pkg_resources
from click_plugins import with_plugins
from bob.extension.scripts.click_helper import AliasedGroup
@with_plugins(pkg_resources.iter_entry_points("bob.bio.demographics.scores.cli"))
@click.group(cls=AliasedGroup)
def score_level_fairness():
"""Score level fairness algorithms"""
pass
# Running a batch of experiments on MOBIO
import os
from bob.extension import rc
import logging
import click
# from bob.extension.scripts.click_helper import ConfigCommand
# from bob.extension.scripts.click_helper import ResourceOption
# from bob.extension.scripts.click_helper import verbosity_option
from bob.bio.face.embeddings.tensorflow import (
facenet_sanderberg_20170512_110547,
resnet50_msceleb_arcface_20210521,
)
from bob.bio.face.embeddings.opencv import vgg16_oxford_baseline
from bob.bio.face.embeddings.pytorch import (
iresnet100,
iresnet50,
AttentionNet,
ResNeSt,
MobileFaceNet,
ResNet,
EfficientNet,
TF_NAS,
HRNet,
ReXNet,
GhostNet,
)
from bob.bio.face.embeddings.mxnet import arcface_insightFace_lresnet100
### Defining baselines
score_level_baselines = dict()
#score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
score_level_baselines["facenet-sanderberg "] = facenet_sanderberg_20170512_110547
#score_level_baselines[
# "resnet50-msceleb-arcface-20210521"
#] = resnet50_msceleb_arcface_20210521
#score_level_baselines["iresnet50"] = iresnet50
#score_level_baselines["attentionnet"] = AttentionNet
#score_level_baselines["resnest"] = ResNeSt
#score_level_baselines["mobilefacenet"] = MobileFaceNet
#score_level_baselines["resnet"] = ResNet
#score_level_baselines["efficientnet"] = EfficientNet
#score_level_baselines["tfnas"] = TF_NAS
#score_level_baselines["hrnet"] = HRNet
#score_level_baselines["rexnet"] = ReXNet
#score_level_baselines["ghostnet"] = GhostNet
#score_level_baselines["vgg16-oxford"] = vgg16_oxford_baseline
def execute(
baseline,
output_path,
database,
sge,
top_norm,
top_norm_score_fraction,
groups=["dev", "eval"],
):
baselines = list(score_level_baselines.keys()) if baseline == "all" else [baseline]
## PLEASE SET
# bob config set bob.bio.demographics.path [OUTPUT-PATH]
output_path = "./results" if output_path is None else output_path
## Running baselines
from bob.bio.base.pipelines.vanilla_biometrics import (
execute_vanilla_biometrics_score_normalization,
)
for b in baselines:
print(f"Running {b}....")
dask_client = "single-threaded"
if sge:
from dask.distributed import Client
from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
from bob.pipelines.distributed.sge_queues import QUEUE_GPU
#cluster = SGEMultipleQueuesCluster(min_jobs=1, sge_job_spec=QUEUE_GPU)
cluster = SGEMultipleQueuesCluster(min_jobs=1)
dask_client = Client(cluster)
### Running the baseline
experiment_path = os.path.join(output_path, "score-norm", database.name, b)
execute_vanilla_biometrics_score_normalization(
score_level_baselines[b](
annotation_type=database.annotation_type,
fixed_positions=database.fixed_positions,
),
database,
dask_client,
groups=groups,
output=experiment_path,
write_metadata_scores=True,
checkpoint=True,
dask_partition_size=100,
dask_n_workers=20,
checkpoint_dir=None,
top_norm=top_norm,
top_norm_score_fraction=top_norm_score_fraction,
score_normalization_type="znorm",
)
"""
execute_vanilla_biometrics_score_normalization(
score_level_baselines[b](
annotation_type=database.annotation_type,
fixed_positions=database.fixed_positions,
),
database,
dask_client,
groups=groups,
output=experiment_path,
write_metadata_scores=True,
checkpoint=True,
dask_partition_size=100,
dask_n_workers=20,
checkpoint_dir=None,
top_norm=top_norm,
top_norm_score_fraction=top_norm_score_fraction,
score_normalization_type="tnorm",
)
"""
if dask_client is not None:
dask_client.shutdown()
default_mobio_protocol = "mobile0-male-female"
@click.command()
@click.option(
"--baseline",
type=click.Choice(
list(score_level_baselines.keys()) + ["all"], case_sensitive=False
),
default="all",
)
@click.option(
"--output-path",
default=rc.get("bob.bio.demographics.path"),
help=f'Output path. Default to: {rc.get("bob.bio.demographics.path")}',
)
@click.option(
"--protocol",
default=default_mobio_protocol,
help=f"Default mobio protocol: {default_mobio_protocol}",
)
@click.option("--sge", is_flag=True)
@click.option(
"--top-norm",
"-t",
is_flag=True,
help="If set, it will do the top-norm.",
)
@click.option(
"--top-norm-score-fraction",
default=1.0,
type=float,
help="Sets the percentage of samples used for t-norm and z-norm. Sometimes you don't want to use all the t/z samples for normalization",
)
def mobio(
baseline, output_path, protocol, sge, top_norm, top_norm_score_fraction, **kwargs
):
"""
Mobio dataset experiments
"""
# Crafting the databse
from bob.bio.face.database import MobioDatabase
database = MobioDatabase(protocol=protocol)
execute(baseline, output_path, database, sge, top_norm, top_norm_score_fraction)
default_meds_protocol = "verification_fold1"
@click.command()
@click.option(
"--baseline",
type=click.Choice(
list(score_level_baselines.keys()) + ["all"], case_sensitive=False
),
default="all",
)
@click.option(
"--output-path",
default=rc.get("bob.bio.demographics.path"),
help=f'Output path. Default to: {rc.get("bob.bio.demographics.path")}',
)
@click.option(
"--protocol",
default=default_meds_protocol,
help=f"Default mobio protocol: {default_meds_protocol}",
)
@click.option("--sge", is_flag=True)
@click.option(
"--top-norm",
"-t",
is_flag=True,
help="If set, it will do the top-norm.",
)
@click.option(
"--top-norm-score-fraction",
default=1.0,
type=float,
help="Sets the percentage of samples used for t-norm and z-norm. Sometimes you don't want to use all the t/z samples for normalization",
)
def meds(
baseline, output_path, protocol, sge, top_norm, top_norm_score_fraction, **kwargs
):
"""
MEDS II dataset experiments
"""
# Crafting the databse
from bob.bio.face.database import MEDSDatabase
database = MEDSDatabase(protocol=protocol)
execute(baseline, output_path, database, sge, top_norm, top_norm_score_fraction)
default_morph_protocol = "verification_fold1"
@click.command()
@click.option(
"--baseline",
type=click.Choice(
list(score_level_baselines.keys()) + ["all"], case_sensitive=False
),
default="all",
)
@click.option(
"--output-path",
default=rc.get("bob.bio.demographics.path"),
help=f'Output path. Default to: {rc.get("bob.bio.demographics.path")}',
)
@click.option(
"--protocol",
default=default_morph_protocol,
help=f"Default mobio protocol: {default_morph_protocol}",
)
@click.option("--sge", is_flag=True)
@click.option(
"--top-norm",
"-t",
is_flag=True,
help="If set, it will do the top-norm.",
)
@click.option(
"--top-norm-score-fraction",
default=1.0,
type=float,
help="Sets the percentage of samples used for t-norm and z-norm. Sometimes you don't want to use all the t/z samples for normalization",
)
def morph(
baseline, output_path, protocol, sge, top_norm, top_norm_score_fraction, **kwargs
):
"""
Morph dataset experiments
"""
# Crafting the databse
from bob.bio.face.database import MorphDatabase
database = MorphDatabase(protocol=protocol)
execute(baseline, output_path, database, sge, top_norm, top_norm_score_fraction)
default_rfw_protocol = "idiap"
@click.command()
@click.option(
"--baseline",
type=click.Choice(
list(score_level_baselines.keys()) + ["all"], case_sensitive=False
),
default="all",
)
@click.option(
"--output-path",
default=rc.get("bob.bio.demographics.path"),
help=f'Output path. Default to: {rc.get("bob.bio.demographics.path")}',
)
@click.option(
"--protocol",
default=default_rfw_protocol,
help=f"Default mobio protocol: {default_rfw_protocol}",
)
@click.option("--sge", is_flag=True)
@click.option(
"--top-norm",
"-t",
is_flag=True,
help="If set, it will do the top-norm.",
)
@click.option(
"--top-norm-score-fraction",
default=1.0,
type=float,
help="Sets the percentage of samples used for t-norm and z-norm. Sometimes you don't want to use all the t/z samples for normalization",
)
def rfw(
baseline, output_path, protocol, sge, top_norm, top_norm_score_fraction, **kwargs
):
"""
Racial faces in the wild experiments
"""
# Crafting the databse
from bob.bio.face.database import RFWDatabase
database = RFWDatabase(protocol=protocol)
execute(
baseline,
output_path,
database,
sge,
top_norm,
top_norm_score_fraction,
groups=["dev"],
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment