Commit e490ef63 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Finished calibration by group

parent 87979a85
......@@ -3,7 +3,17 @@ from bob.bio.base.pipelines.vanilla_biometrics import (
CategoricalCalibration,
WeibullCalibration,
LLRCalibration,
GammaCalibration,
)
import numpy as np
def _arg_split(ctx, param, value):
# split columns by ',' and remove whitespace
return [c.strip() for c in value.split(",")]
calibration_types = ["linear", "weibull", "gamma"]
def calibrate(
......@@ -11,7 +21,7 @@ def calibrate(
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
score_selection_method,
field_name,
field_values,
):
......@@ -19,19 +29,22 @@ def calibrate(
calibrator = LLRCalibration
elif calibrator == "weibull":
calibrator = WeibullCalibration
elif calibrator == "gamma":
calibrator = GammaCalibration
else:
raise ValueError(f"Invalid calibrator {calibrator}")
if quantile_factor=="none":
quantile_factor = None
else:
quantile_factor = float(quantile_factor)
# if quantile_factor == "none":
# quantile_factor = None
# else:
# quantile_factor = float(quantile_factor)
calibrator = CategoricalCalibration(
field_name=field_name,
field_values=field_values,
quantile_factor=quantile_factor,
score_selection_method=score_selection_method,
fit_estimator=calibrator,
reduction_function=np.max,
)
calibrator = calibrator.fit(fit_score_file)
calibrator.transform(transform_score_files, output_score_files)
......@@ -42,49 +55,212 @@ def calibrate(
@click.option(
"--transform-score-files",
required=True,
multiple=True,
help="List of scores to be calibrated",
help="List of scores to be calibrated (comma separated)",
callback=_arg_split,
)
@click.option(
"--output-score-files",
required=True,
multiple=True,
help="Output scores.",
help="Output scores (comma separated).",
callback=_arg_split,
)
@click.option(
"--calibrator",
type=click.Choice(["linear", "weibull"], case_sensitive=False),
type=click.Choice(calibration_types, case_sensitive=False),
default="linear",
help="Calibrators.",
)
@click.option(
"--quantile-factor",
default="1.5",
help="Quantile factor used to pic the scores that fits the calibrator",
"--score-selection-method",
default="q3-median",
help="Method to select the scores for fetting the calibration models."
"`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
"`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
"`all`: It will select all the scores."
" Default to `median-q3`",
)
def mobio(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
score_selection_method,
):
"""
Calibrates scores coming from experiments using MOBIO database.
"""
#field_values = ["Asian", "African", "Indian", "Caucasian"]
#field_name = "race"
field_values = ["m", "f"]
field_name = "gender"
calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
field_name,
field_values,
)
@click.command()
@click.argument("fit-score-file")
@click.option(
"--transform-score-files",
required=True,
help="List of scores to be calibrated (comma separated)",
callback=_arg_split,
)
@click.option(
"--output-score-files",
required=True,
help="Output scores (comma separated).",
callback=_arg_split,
)
@click.option(
"--calibrator",
type=click.Choice(calibration_types, case_sensitive=False),
default="linear",
help="Calibrators.",
)
@click.option(
"--score-selection-method",
default="q3-median",
help="Method to select the scores for fetting the calibration models."
"`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
"`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
"`all`: It will select all the scores."
" Default to `median-q3`",
)
def meds(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
):
"""
Calibrates scores coming from experiments using MEDS database.
"""
field_values = ["B", "W"]
field_name = "rac"
calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
field_name,
field_values,
)
@click.command()
@click.argument("fit-score-file")
@click.option(
"--transform-score-files",
required=True,
help="List of scores to be calibrated (comma separated)",
callback=_arg_split,
)
@click.option(
"--output-score-files",
required=True,
help="Output scores (comma separated).",
callback=_arg_split,
)
@click.option(
"--calibrator",
type=click.Choice(calibration_types, case_sensitive=False),
default="linear",
help="Calibrators.",
)
@click.option(
"--score-selection-method",
default="q3-median",
help="Method to select the scores for fetting the calibration models."
"`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
"`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
"`all`: It will select all the scores."
" Default to `median-q3`",
)
def rfw(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
):
"""
Calibrates scores coming from experiments using RFW database.
"""
field_values = ["Asian", "African", "Indian", "Caucasian"]
field_name = "race"
calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
field_name,
field_values,
)
@click.command()
@click.argument("fit-score-file")
@click.option(
"--transform-score-files",
required=True,
help="List of scores to be calibrated (comma separated)",
callback=_arg_split,
)
@click.option(
"--output-score-files",
required=True,
help="Output scores (comma separated).",
callback=_arg_split,
)
@click.option(
"--calibrator",
type=click.Choice(calibration_types, case_sensitive=False),
default="linear",
help="Calibrators.",
)
@click.option(
"--score-selection-method",
default="q3-median",
help="Method to select the scores for fetting the calibration models."
"`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
"`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
"`all`: It will select all the scores."
" Default to `median-q3`",
)
def morph_race(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
score_selection_method,
):
"""
Calibrates scores coming from experiments using RFW database.
"""
field_values = ["A", "W", "B", "H"]
field_name = "rac"
calibrate(
fit_score_file,
transform_score_files,
output_score_files,
calibrator,
quantile_factor,
score_selection_method,
field_name,
field_values,
)
......@@ -39,22 +39,22 @@ from bob.bio.face.embeddings.mxnet import arcface_insightFace_lresnet100
### Defining baselines
score_level_baselines = dict()
#score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
score_level_baselines["facenet-sanderberg "] = facenet_sanderberg_20170512_110547
#score_level_baselines[
# "resnet50-msceleb-arcface-20210521"
#] = resnet50_msceleb_arcface_20210521
#score_level_baselines["iresnet50"] = iresnet50
#score_level_baselines["attentionnet"] = AttentionNet
#score_level_baselines["resnest"] = ResNeSt
#score_level_baselines["mobilefacenet"] = MobileFaceNet
#score_level_baselines["resnet"] = ResNet
#score_level_baselines["efficientnet"] = EfficientNet
#score_level_baselines["tfnas"] = TF_NAS
#score_level_baselines["hrnet"] = HRNet
#score_level_baselines["rexnet"] = ReXNet
#score_level_baselines["ghostnet"] = GhostNet
score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
#score_level_baselines["facenet-sanderberg "] = facenet_sanderberg_20170512_110547
score_level_baselines[
"resnet50-msceleb-arcface-20210521"
] = resnet50_msceleb_arcface_20210521
score_level_baselines["iresnet50"] = iresnet50
score_level_baselines["attentionnet"] = AttentionNet
score_level_baselines["resnest"] = ResNeSt
score_level_baselines["mobilefacenet"] = MobileFaceNet
score_level_baselines["resnet"] = ResNet
score_level_baselines["efficientnet"] = EfficientNet
score_level_baselines["tfnas"] = TF_NAS
score_level_baselines["hrnet"] = HRNet
score_level_baselines["rexnet"] = ReXNet
score_level_baselines["ghostnet"] = GhostNet
#score_level_baselines["vgg16-oxford"] = vgg16_oxford_baseline
......
......@@ -54,7 +54,10 @@ setup(
"rfw = bob.bio.demographics.script.commands:rfw ",
],
"bob.bio.demographics.calibration.cli": [
"meds = bob.bio.demographics.script.calibration_commands:meds",
"mobio = bob.bio.demographics.script.calibration_commands:mobio",
"rfw = bob.bio.demographics.script.calibration_commands:rfw",
"morph-race = bob.bio.demographics.script.calibration_commands:morph_race",
],
},
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment