Finished calibration by group

bob/bio/demographics/script/calibration_commands.py

@@ -3,7 +3,17 @@ from bob.bio.base.pipelines.vanilla_biometrics import (
     CategoricalCalibration,
     WeibullCalibration,
     LLRCalibration,
+    GammaCalibration,
 )
+import numpy as np
+
+
+def _arg_split(ctx, param, value):
+    # split columns by ',' and remove whitespace
+    return [c.strip() for c in value.split(",")]
+
+
+calibration_types = ["linear", "weibull", "gamma"]
 
 
 def calibrate(
@@ -11,7 +21,7 @@ def calibrate(
     transform_score_files,
     output_score_files,
     calibrator,
-    quantile_factor,
+    score_selection_method,
     field_name,
     field_values,
 ):
@@ -19,19 +29,22 @@ def calibrate(
         calibrator = LLRCalibration
     elif calibrator == "weibull":
         calibrator = WeibullCalibration
+    elif calibrator == "gamma":
+        calibrator = GammaCalibration
     else:
         raise ValueError(f"Invalid calibrator {calibrator}")
 
-    if quantile_factor=="none":
-        quantile_factor = None
-    else:
-        quantile_factor = float(quantile_factor)
+    # if quantile_factor == "none":
+    #    quantile_factor = None
+    # else:
+    #    quantile_factor = float(quantile_factor)
 
     calibrator = CategoricalCalibration(
         field_name=field_name,
         field_values=field_values,
-        quantile_factor=quantile_factor,
+        score_selection_method=score_selection_method,
         fit_estimator=calibrator,
+        reduction_function=np.max,
     )
     calibrator = calibrator.fit(fit_score_file)
     calibrator.transform(transform_score_files, output_score_files)
@@ -42,49 +55,212 @@ def calibrate(
 @click.option(
     "--transform-score-files",
     required=True,
-    multiple=True,
-    help="List of scores to be calibrated",
+    help="List of scores to be calibrated (comma separated)",
+    callback=_arg_split,
 )
 @click.option(
     "--output-score-files",
     required=True,
-    multiple=True,
-    help="Output scores.",
+    help="Output scores (comma separated).",
+    callback=_arg_split,
 )
 @click.option(
     "--calibrator",
-    type=click.Choice(["linear", "weibull"], case_sensitive=False),
+    type=click.Choice(calibration_types, case_sensitive=False),
     default="linear",
     help="Calibrators.",
 )
 @click.option(
-    "--quantile-factor",
-    default="1.5",
-    help="Quantile factor used to pic the scores that fits the calibrator",
+    "--score-selection-method",
+    default="q3-median",
+    help="Method to select the scores for fetting the calibration models."
+    "`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
+    "`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
+    "`all`: It will select all the scores."
+    " Default to `median-q3`",
 )
 def mobio(
     fit_score_file,
     transform_score_files,
     output_score_files,
     calibrator,
-    quantile_factor,
+    score_selection_method,
 ):
     """
     Calibrates scores coming from experiments using MOBIO database.
     """
 
-    #field_values = ["Asian", "African", "Indian", "Caucasian"]
-    #field_name = "race"
     field_values = ["m", "f"]
     field_name = "gender"
 
+    calibrate(
+        fit_score_file,
+        transform_score_files,
+        output_score_files,
+        calibrator,
+        score_selection_method,
+        field_name,
+        field_values,
+    )
+
+
+@click.command()
+@click.argument("fit-score-file")
+@click.option(
+    "--transform-score-files",
+    required=True,
+    help="List of scores to be calibrated (comma separated)",
+    callback=_arg_split,
+)
+@click.option(
+    "--output-score-files",
+    required=True,
+    help="Output scores (comma separated).",
+    callback=_arg_split,
+)
+@click.option(
+    "--calibrator",
+    type=click.Choice(calibration_types, case_sensitive=False),
+    default="linear",
+    help="Calibrators.",
+)
+@click.option(
+    "--score-selection-method",
+    default="q3-median",
+    help="Method to select the scores for fetting the calibration models."
+    "`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
+    "`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
+    "`all`: It will select all the scores."
+    " Default to `median-q3`",
+)
+def meds(
+    fit_score_file,
+    transform_score_files,
+    output_score_files,
+    calibrator,
+    score_selection_method,
+):
+    """
+    Calibrates scores coming from experiments using MEDS database.
+    """
+
+    field_values = ["B", "W"]
+    field_name = "rac"
+
+    calibrate(
+        fit_score_file,
+        transform_score_files,
+        output_score_files,
+        calibrator,
+        score_selection_method,
+        field_name,
+        field_values,
+    )
+
+
+@click.command()
+@click.argument("fit-score-file")
+@click.option(
+    "--transform-score-files",
+    required=True,
+    help="List of scores to be calibrated (comma separated)",
+    callback=_arg_split,
+)
+@click.option(
+    "--output-score-files",
+    required=True,
+    help="Output scores (comma separated).",
+    callback=_arg_split,
+)
+@click.option(
+    "--calibrator",
+    type=click.Choice(calibration_types, case_sensitive=False),
+    default="linear",
+    help="Calibrators.",
+)
+@click.option(
+    "--score-selection-method",
+    default="q3-median",
+    help="Method to select the scores for fetting the calibration models."
+    "`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
+    "`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
+    "`all`: It will select all the scores."
+    " Default to `median-q3`",
+)
+def rfw(
+    fit_score_file,
+    transform_score_files,
+    output_score_files,
+    calibrator,
+    score_selection_method,
+):
+    """
+    Calibrates scores coming from experiments using RFW database.
+    """
+
+    field_values = ["Asian", "African", "Indian", "Caucasian"]
+    field_name = "race"
+
+    calibrate(
+        fit_score_file,
+        transform_score_files,
+        output_score_files,
+        calibrator,
+        score_selection_method,
+        field_name,
+        field_values,
+    )
+
+
+@click.command()
+@click.argument("fit-score-file")
+@click.option(
+    "--transform-score-files",
+    required=True,
+    help="List of scores to be calibrated (comma separated)",
+    callback=_arg_split,
+)
+@click.option(
+    "--output-score-files",
+    required=True,
+    help="Output scores (comma separated).",
+    callback=_arg_split,
+)
+@click.option(
+    "--calibrator",
+    type=click.Choice(calibration_types, case_sensitive=False),
+    default="linear",
+    help="Calibrators.",
+)
+@click.option(
+    "--score-selection-method",
+    default="q3-median",
+    help="Method to select the scores for fetting the calibration models."
+    "`median-q3`: It will select the scores from the median to q3 from the impostor scores (q1 to median for genuines)"
+    "`q3-outlier`: It will select the scores from q3 to outlier (q3+1.5*IQD) from the impostor scores (q1 to outlier for genuines)"
+    "`all`: It will select all the scores."
+    " Default to `median-q3`",
+)
+def morph_race(
+    fit_score_file,
+    transform_score_files,
+    output_score_files,
+    calibrator,
+    score_selection_method,
+):
+    """
+    Calibrates scores coming from experiments using RFW database.
+    """
+
+    field_values = ["A", "W", "B", "H"]
+    field_name = "rac"
 
     calibrate(
         fit_score_file,
         transform_score_files,
         output_score_files,
         calibrator,
-        quantile_factor,
+        score_selection_method,
         field_name,
         field_values,
     )

bob/bio/demographics/script/score_level_fairness_commands.py

@@ -39,22 +39,22 @@ from bob.bio.face.embeddings.mxnet import arcface_insightFace_lresnet100
 
 ### Defining baselines
 score_level_baselines = dict()
-#score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
-score_level_baselines["facenet-sanderberg "] = facenet_sanderberg_20170512_110547
-#score_level_baselines[
-#    "resnet50-msceleb-arcface-20210521"
-#] = resnet50_msceleb_arcface_20210521
-#score_level_baselines["iresnet50"] = iresnet50
-#score_level_baselines["attentionnet"] = AttentionNet
-
-#score_level_baselines["resnest"] = ResNeSt
-#score_level_baselines["mobilefacenet"] = MobileFaceNet
-#score_level_baselines["resnet"] = ResNet
-#score_level_baselines["efficientnet"] = EfficientNet
-#score_level_baselines["tfnas"] = TF_NAS
-#score_level_baselines["hrnet"] = HRNet
-#score_level_baselines["rexnet"] = ReXNet
-#score_level_baselines["ghostnet"] = GhostNet
+score_level_baselines["arcface-insightface"] = arcface_insightFace_lresnet100
+#score_level_baselines["facenet-sanderberg "] = facenet_sanderberg_20170512_110547
+score_level_baselines[
+    "resnet50-msceleb-arcface-20210521"
+] = resnet50_msceleb_arcface_20210521
+score_level_baselines["iresnet50"] = iresnet50
+score_level_baselines["attentionnet"] = AttentionNet
+
+score_level_baselines["resnest"] = ResNeSt
+score_level_baselines["mobilefacenet"] = MobileFaceNet
+score_level_baselines["resnet"] = ResNet
+score_level_baselines["efficientnet"] = EfficientNet
+score_level_baselines["tfnas"] = TF_NAS
+score_level_baselines["hrnet"] = HRNet
+score_level_baselines["rexnet"] = ReXNet
+score_level_baselines["ghostnet"] = GhostNet
 #score_level_baselines["vgg16-oxford"] = vgg16_oxford_baseline
 
 

setup.py

@@ -54,7 +54,10 @@ setup(
             "rfw   = bob.bio.demographics.script.commands:rfw ",
         ],
         "bob.bio.demographics.calibration.cli": [
+            "meds          = bob.bio.demographics.script.calibration_commands:meds",
             "mobio          = bob.bio.demographics.script.calibration_commands:mobio",
+            "rfw          = bob.bio.demographics.script.calibration_commands:rfw",
+            "morph-race          = bob.bio.demographics.script.calibration_commands:morph_race",
         ],
 
     },