Commit d02339d3 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Update

parent 35a813df
Pipeline #53288 failed with stages
in 9 minutes and 48 seconds
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from .scores import (
compute_fmr_thresholds,
split_scores_by_variable,
......
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
......@@ -18,7 +18,8 @@ The analysis on some datasets can be found in::
.. toctree::
meds/meds
meds/index
mobio/index
......@@ -9,11 +9,11 @@
Follow below the demographic differentials considering some FR baselines using the :ref:`Fairness Discrepancy Rate <bob.bio.demographics.fdr>` .
.. figure:: meds/meds.png
.. figure:: meds.png
:align: center
:alt: Fairness Discrepancy Rate
FDR and FMR/FNMR trade-offs (:download:`pdf <meds/meds.pdf>`)
FDR and FMR/FNMR trade-offs (:download:`pdf <meds.pdf>`)
......@@ -150,6 +150,7 @@ Black-Black 0 0 0 0.002 0.024 0.183
Black-White 0 0 0 0 0.001 0.047
=========== ===== ===== ====== ===== ===== =====
FNMR
====
......
.. =*= coding: utf=8 =*=
.. _bob.bio.demographics.meds:
========
MEDS II
========
Follow below the demographic differentials considering some FR baselines.
Facenet-sanderberg
------------------
`Facenet <https://github.com/davidsandberg/facenet>`_ model.
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0 0.019
White-White 0 0 0 0 0.004 0.067
Black-Black 0 0 0 0.002 0.032 0.289
Black-White 0 0 0 0 0 0.017
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ===== ===
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.131 0.131 0.119 0.06 0.012 0
Black-Black 0.08 0.08 0.08 0.013 0 0
=========== ===== ===== ====== ===== ===== ===
iResnet 100
-----------
`Arcface (iResnet 101) model <https://github.com/deepinsight/insightface>`_ ported to pytorch.
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0.003 0.065
White-White 0 0 0 0 0.01 0.117
Black-Black 0 0 0 0.002 0.019 0.144
Black-White 0 0 0 0 0.003 0.071
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ==== ===
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.012 0.012 0 0 0 0
Black-Black 0.004 0.004 0 0 0 0
=========== ===== ===== ====== ===== ==== ===
iResnet 50
----------
`Arcface (iResnet 50) model <https://github.com/deepinsight/insightface>`_ ported to pytorch.
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0.002 0.051
White-White 0 0 0 0.001 0.008 0.109
Black-Black 0 0 0 0.002 0.022 0.161
Black-White 0 0 0 0 0.002 0.056
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.071 0.071 0.06 0.06 0.036 0.024
Black-Black 0.093 0.093 0.089 0.04 0.009 0
=========== ===== ===== ====== ===== ===== =====
idiap_resnet50_arcface_2021
---------------------------
Arcface (resnet 50 **not the iResnet**) model trained using tensorflow.
FNMR
====
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0.003 0.047
White-White 0 0 0 0 0.011 0.124
Black-Black 0 0 0.001 0.003 0.022 0.153
Black-White 0 0 0 0 0.002 0.042
=========== ===== ===== ====== ===== ===== =====
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.143 0.143 0.131 0.095 0.036 0
Black-Black 0.16 0.16 0.133 0.093 0.04 0.004
=========== ===== ===== ====== ===== ===== =====
idiap_resnet50_arcface_20210521
-------------------------------
Arcface (resnet 50 **not the iResnet**) model trained using tensorflow.
This one is more recent.
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0.001 0.039
White-White 0 0 0 0.001 0.01 0.118
Black-Black 0 0 0 0.002 0.024 0.183
Black-White 0 0 0 0 0.001 0.047
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.155 0.155 0.143 0.06 0.036 0.012
Black-Black 0.116 0.116 0.107 0.049 0.022 0
=========== ===== ===== ====== ===== ===== =====
VGG - 16
--------
VGG 16 Face model
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0 0.012
White-White 0 0 0 0 0.002 0.033
Black-Black 0 0 0.001 0.004 0.04 0.306
Black-White 0 0 0 0 0 0.012
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.56 0.56 0.381 0.214 0.119 0.024
Black-Black 0.551 0.551 0.356 0.2 0.093 0.009
=========== ===== ===== ====== ===== ===== =====
arcface_insightface
-------------------
Original `Arcface (iResnet 101) model <https://github.com/deepinsight/insightface>`_.
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0.003 0.065
White-White 0 0 0 0 0.01 0.117
Black-Black 0 0 0 0.002 0.019 0.144
Black-White 0 0 0 0 0.003 0.071
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ==== ===
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.012 0.012 0 0 0 0
Black-Black 0.004 0.004 0 0 0 0
=========== ===== ===== ====== ===== ==== ===
idiap_inc_res_v2_2018
-------------------------------------------
Inception-resnet v2 model trained in [TFP18]_:
FMR
===
=========== ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-Black 0 0 0 0 0 0.027
White-White 0 0 0 0 0.006 0.09
Black-Black 0 0 0 0.003 0.028 0.236
Black-White 0 0 0 0 0 0.023
=========== ===== ===== ====== ===== ===== =====
FNMR
====
=========== ===== ===== ====== ===== ===== ===
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
White-White 0.155 0.155 0.131 0.06 0.012 0
Black-Black 0.147 0.147 0.111 0.027 0.004 0
=========== ===== ===== ====== ===== ===== ===
.. =*= coding: utf=8 =*=
.. _bob.bio.demographics.mobio.analysis:
=========
MOBIO II
=========
Follow below the demographic differentials considering some FR baselines using the :ref:`Fairness Discrepancy Rate <bob.bio.demographics.fdr>` .
.. figure:: mobio.png
:align: center
:alt: Fairness Discrepancy Rate
FDR and FMR/FNMR trade-offs (:download:`pdf <mobio.pdf>`)
Facenet-sanderberg
------------------
`Facenet <https://github.com/davidsandberg/facenet>`_ model.
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.003 0.038 0.281
Female-Male 0 0 0 0 0.001 0.026
Male-Female 0 0 0 0 0.001 0.031
Male-Male 0 0 0 0 0.005 0.081
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.127 0.127 0.068 0.026 0.003 0
Male-Male 0.033 0.033 0.016 0.006 0.002 0.001
============= ===== ===== ====== ===== ===== =====
iResnet 100
-----------
`Arcface (iResnet 101) model <https://github.com/deepinsight/insightface>`_ ported to pytorch.
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.004 0.027 0.235
Female-Male 0 0 0 0 0.001 0.043
Male-Female 0 0 0 0 0.001 0.052
Male-Male 0 0 0 0 0.006 0.077
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0 0 0
Male-Male 0.002 0.002 0.001 0.001 0.001 0.001
============= ===== ===== ====== ===== ===== =====
iResnet 50
----------
`Arcface (iResnet 50) model <https://github.com/deepinsight/insightface>`_ ported to pytorch.
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.003 0.046 0.282
Female-Male 0 0 0 0 0.002 0.036
Male-Female 0 0 0 0 0.002 0.045
Male-Male 0 0 0 0 0.004 0.069
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.098 0.098 0.032 0.008 0 0
Male-Male 0.109 0.109 0.041 0.014 0.004 0.002
============= ===== ===== ====== ===== ===== =====
idiap_resnet50_arcface_2021
---------------------------
Arcface (resnet 50 **not the iResnet**) model trained using tensorflow.
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.003 0.058 0.297
Female-Male 0 0 0 0 0.001 0.033
Male-Female 0 0 0 0 0.001 0.046
Male-Male 0 0 0 0 0.005 0.075
============= ===== ===== ====== ===== ===== =====
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.525 0.525 0.255 0.077 0.015 0.001
Male-Male 0.403 0.403 0.208 0.07 0.011 0.002
============= ===== ===== ====== ===== ===== =====
idiap_resnet50_arcface_20210521
-------------------------------
Arcface (resnet 50 **not the iResnet**) model trained using tensorflow.
This one is more recent.
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.001 0.021 0.297
Female-Male 0 0 0 0 0 0.026
Male-Female 0 0 0 0 0 0.036
Male-Male 0 0 0 0 0.004 0.075
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.29 0.29 0.116 0.05 0.006 0
Male-Male 0.192 0.192 0.095 0.039 0.005 0.001
============= ===== ===== ====== ===== ===== =====
VGG - 16
--------
VGG 16 Face model
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0 0.004 0.056
Female-Male 0 0 0 0 0 0.011
Male-Female 0 0 0 0 0 0.006
Male-Male 0 0 0 0 0 0.009
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.848 0.848 0.65 0.48 0.25 0.059
Male-Male 0.788 0.788 0.578 0.417 0.182 0.03
============= ===== ===== ====== ===== ===== =====
arcface_insightface
-------------------
Original `Arcface (iResnet 101) model <https://github.com/deepinsight/insightface>`_.
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0.004 0.027 0.235
Female-Male 0 0 0 0 0.001 0.043
Male-Female 0 0 0 0 0.001 0.052
Male-Male 0 0 0 0 0.006 0.077
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0 0 0 0
Male-Male 0.002 0.002 0.001 0.001 0.001 0.001
============= ===== ===== ====== ===== ===== =====
idiap_inc_res_v2_2018
-------------------------------------------
Inception-resnet v2 model trained in [TFP18]_:
FMR
===
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0 0 0.001 0.004 0.03 0.282
Female-Male 0 0 0 0 0 0.011
Male-Female 0 0 0 0 0 0.014
Male-Male 0 0 0 0 0.002 0.068
============= ===== ===== ====== ===== ===== =====
FNMR
====
============= ===== ===== ====== ===== ===== =====
Ethnicities 1e-06 1e-05 0.0001 0.001 0.01 0.1
Female-Female 0.209 0.209 0.09 0.044 0.01 0
Male-Male 0.106 0.106 0.037 0.015 0.003 0.001
============= ===== ===== ====== ===== ===== =====
# Running a batch of experiments on MEDS
import os
from bob.extension import rc
import click
@click.command()
@click.option(
"--output-path",
default=rc.get("bob.bio.demographics.path"),
help=f"Output path. Default to: {rc.get('bob.bio.demographics.path')}",
)
@click.option(
"--protocol", default="original", help=f"RFW protocol. `original` or `idiap`",
)
@click.option("--sge", is_flag=True)
def run(output_path, protocol, sge):
consider_genuines = False
ztnorm_cohort_proportion = 1.0
# DATABASE
from bob.bio.face.database import RFWDatabase
protocol = protocol
database = RFWDatabase(protocol=protocol)
database_name = "rfw"
# Baselines
from bob.bio.face.embeddings.tensorflow import (
inception_resnet_v2_msceleb_centerloss_2018,
facenet_sanderberg_20170512_110547,
resnet50_msceleb_arcface_2021,
resnet50_msceleb_arcface_20210521,
)
from bob.bio.face.embeddings.pytorch import (
iresnet100,
iresnet50,
)
from bob.bio.face.embeddings.mxnet import arcface_insightFace_lresnet100
from bob.bio.face.embeddings.opencv import vgg16_oxford_baseline
baselines = [
inception_resnet_v2_msceleb_centerloss_2018,
facenet_sanderberg_20170512_110547,
resnet50_msceleb_arcface_2021,
resnet50_msceleb_arcface_20210521,
iresnet100,
iresnet50,
arcface_insightFace_lresnet100,
vgg16_oxford_baseline,
]
baselines = [resnet50_msceleb_arcface_20210521]
## Running baselines
from bob.bio.base.pipelines.vanilla_biometrics import (
execute_vanilla_biometrics_ztnorm,
)
for b in baselines:
print(f"Running {b.__name__}....")
dask_client = None
if sge:
from dask.distributed import Client
from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
cluster = SGEMultipleQueuesCluster(min_jobs=1)
dask_client = Client(cluster)
### Running the baseline
experiment_path = os.path.join(
output_path, "score-norm", database_name, b.__name__
)
execute_vanilla_biometrics_ztnorm(
b(
annotation_type=database.annotation_type,
fixed_positions=database.fixed_positions,
),
database,
dask_client,
groups=["dev"],
output=experiment_path,
write_metadata_scores=True,
checkpoint=True,
dask_partition_size=300,
dask_n_workers=20,