Merge branch 'tf-arcface' into 'master'

New baselines

See merge request !102

bob/bio/face/config/baseline/helpers.py

@@ -235,7 +235,11 @@ def embedding_transformer_112x112(
     cropped_image_size = (112, 112)
     if annotation_type == "eyes-center":
         # Hard coding eye positions for backward consistency
-        cropped_positions = {"leye": (49, 72), "reye": (49, 38)}
+        cropped_positions = {
+            "leye": (55, 81),
+            "reye": (55, 42),
+        }
+
     else:
         # Will use default
         cropped_positions = embedding_transformer_default_cropping(

bob/bio/face/config/baseline/mobilenetv2_msceleb_arcface_2021.py

+from bob.bio.face.embeddings.mobilenet_v2 import MobileNetv2_MsCeleb_ArcFace_2021
+from bob.bio.face.config.baseline.helpers import embedding_transformer_112x112
+from bob.bio.base.pipelines.vanilla_biometrics import (
+    Distance,
+    VanillaBiometricsPipeline,
+)
+
+memory_demanding = False
+if "database" in locals():
+    annotation_type = database.annotation_type
+    fixed_positions = database.fixed_positions
+
+    memory_demanding = (
+        database.memory_demanding if hasattr(database, "memory_demanding") else False
+    )
+else:
+    annotation_type = None
+    fixed_positions = None
+
+
+def load(annotation_type, fixed_positions=None):
+    transformer = embedding_transformer_112x112(
+        MobileNetv2_MsCeleb_ArcFace_2021(memory_demanding=memory_demanding),
+        annotation_type,
+        fixed_positions,
+    )
+
+    algorithm = Distance()
+
+    return VanillaBiometricsPipeline(transformer, algorithm)
+
+
+pipeline = load(annotation_type, fixed_positions)
+transformer = pipeline.transformer

bob/bio/face/config/baseline/resnet50_msceleb_arcface_2021.py

+from bob.bio.face.embeddings.resnet50 import Resnet50_MsCeleb_ArcFace_2021
+from bob.bio.face.config.baseline.helpers import embedding_transformer_112x112
+from bob.bio.base.pipelines.vanilla_biometrics import (
+    Distance,
+    VanillaBiometricsPipeline,
+)
+
+memory_demanding = False
+if "database" in locals():
+    annotation_type = database.annotation_type
+    fixed_positions = database.fixed_positions
+
+    memory_demanding = (
+        database.memory_demanding if hasattr(database, "memory_demanding") else False
+    )
+else:
+    annotation_type = None
+    fixed_positions = None
+
+
+def load(annotation_type, fixed_positions=None):
+    transformer = embedding_transformer_112x112(
+        Resnet50_MsCeleb_ArcFace_2021(memory_demanding=memory_demanding),
+        annotation_type,
+        fixed_positions,
+    )
+
+    algorithm = Distance()
+
+    return VanillaBiometricsPipeline(transformer, algorithm)
+
+
+pipeline = load(annotation_type, fixed_positions)
+transformer = pipeline.transformer

bob/bio/face/embeddings/mobilenet_v2.py

+from bob.bio.face.embeddings import download_model
+
+
+from .tf2_inception_resnet import TransformTensorflow
+import pkg_resources
+import os
+from bob.extension import rc
+import tensorflow as tf
+
+
+class MobileNetv2_MsCeleb_ArcFace_2021(TransformTensorflow):
+    """
+    MobileNet Backbone trained with the MSCeleb 1M database.
+
+    The bottleneck layer (a.k.a embedding) has 512d.
+
+    The configuration file used to trained is:
+
+    ```yaml
+    batch-size: 128
+    face-size: 112
+    face-output_size: 112
+    n-classes: 85742
+
+
+    ## Backbone
+    backbone: 'mobilenet-v2'
+    head: 'arcface'
+    s: 10
+    bottleneck: 512
+    m: 0.5
+
+    # Training parameters
+    solver: "sgd"
+    lr: 0.01
+    dropout-rate: 0.5
+    epochs: 500
+
+
+    train-tf-record-path: "<PATH>"
+    validation-tf-record-path: "<PATH>"
+
+    ```
+
+
+    """
+
+    def __init__(self, memory_demanding=False):
+        internal_path = pkg_resources.resource_filename(
+            __name__, os.path.join("data", "mobilenet-v2-msceleb-arcface-2021"),
+        )
+
+        checkpoint_path = (
+            internal_path
+            if rc["bob.bio.face.models.mobilenet-v2-msceleb-arcface-2021"] is None
+            else rc["bob.bio.face.models.mobilenet-v2-msceleb-arcface-2021"]
+        )
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021.tar.gz",
+        ]
+
+        download_model(checkpoint_path, urls, "mobilenet-v2-msceleb-arcface-2021.tar.gz")
+
+        super(MobileNetv2_MsCeleb_ArcFace_2021, self).__init__(
+            checkpoint_path,
+            preprocessor=lambda X: X / 255.0,
+            memory_demanding=memory_demanding,
+        )
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)[0]
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+

bob/bio/face/embeddings/resnet50.py

+from bob.bio.face.embeddings import download_model
+
+
+from .tf2_inception_resnet import TransformTensorflow
+import pkg_resources
+import os
+from bob.extension import rc
+import tensorflow as tf
+
+
+class Resnet50_MsCeleb_ArcFace_2021(TransformTensorflow):
+    """
+    Resnet50 Backbone trained with the MSCeleb 1M database.
+
+    The bottleneck layer (a.k.a embedding) has 512d.
+
+    The configuration file used to trained is:
+
+    ```yaml
+    batch-size: 128
+    face-size: 112
+    face-output_size: 112
+    n-classes: 85742
+
+
+    ## Backbone
+    backbone: 'resnet50'
+    head: 'arcface'
+    s: 10
+    bottleneck: 512
+    m: 0.5
+
+    # Training parameters
+    solver: "sgd"
+    lr: 0.01
+    dropout-rate: 0.5
+    epochs: 500
+
+
+    train-tf-record-path: "<PATH>"
+    validation-tf-record-path: "<PATH>"
+
+    ```
+
+
+    """
+
+    def __init__(self, memory_demanding=False):
+        internal_path = pkg_resources.resource_filename(
+            __name__, os.path.join("data", "resnet50_msceleb_arcface_2021"),
+        )
+
+        checkpoint_path = (
+            internal_path
+            if rc["bob.bio.face.models.resnet50_msceleb_arcface_2021"] is None
+            else rc["bob.bio.face.models.resnet50_msceleb_arcface_2021"]
+        )
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
+        ]
+
+        download_model(checkpoint_path, urls, "resnet50_msceleb_arcface_2021.tar.gz")
+
+        super(Resnet50_MsCeleb_ArcFace_2021, self).__init__(
+            checkpoint_path,
+            preprocessor=lambda X: X / 255.0,
+            memory_demanding=memory_demanding,
+        )
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)[0]
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+

bob/bio/face/embeddings/tf2_inception_resnet.py

@@ -20,9 +20,9 @@ def sanderberg_rescaling():
     return preprocessor
 
 
-class InceptionResnet(TransformerMixin, BaseEstimator):
+class TransformTensorflow(TransformerMixin, BaseEstimator):
     """
-    Base Transformer for InceptionResnet architectures.
+    Base Transformer for Tensorflow architectures.
 
     Szegedy, Christian, et al. "Inception-v4, inception-resnet and the impact of residual connections on learning." arXiv preprint arXiv:1602.07261 (2016).
 
@@ -52,14 +52,6 @@ class InceptionResnet(TransformerMixin, BaseEstimator):
     def load_model(self):
         self.model = tf.keras.models.load_model(self.checkpoint_path)
 
-    def inference(self, X):
-        if self.preprocessor is not None:
-            X = self.preprocessor(tf.cast(X, "float32"))
-
-        prelogits = self.model.predict_on_batch(X)
-        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
-        return embeddings
-
     def transform(self, X):
         def _transform(X):
             X = tf.convert_to_tensor(X)
@@ -88,6 +80,14 @@ class InceptionResnet(TransformerMixin, BaseEstimator):
         d["model"] = None
         return d
 
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+
     def _more_tags(self):
         return {"stateless": True, "requires_fit": False}
 
@@ -95,7 +95,7 @@ class InceptionResnet(TransformerMixin, BaseEstimator):
         self.model = None
 
 
-class InceptionResnetv2_MsCeleb_CenterLoss_2018(InceptionResnet):
+class InceptionResnetv2_MsCeleb_CenterLoss_2018(TransformTensorflow):
     """
     InceptionResnet v2 model trained in 2018 using the MSCeleb dataset in the context of the work:
 
@@ -131,7 +131,7 @@ class InceptionResnetv2_MsCeleb_CenterLoss_2018(InceptionResnet):
         )
 
 
-class InceptionResnetv2_Casia_CenterLoss_2018(InceptionResnet):
+class InceptionResnetv2_Casia_CenterLoss_2018(TransformTensorflow):
     """
     InceptionResnet v2 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
 
@@ -166,7 +166,7 @@ class InceptionResnetv2_Casia_CenterLoss_2018(InceptionResnet):
         )
 
 
-class InceptionResnetv1_Casia_CenterLoss_2018(InceptionResnet):
+class InceptionResnetv1_Casia_CenterLoss_2018(TransformTensorflow):
     """
     InceptionResnet v1 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
 
@@ -201,7 +201,7 @@ class InceptionResnetv1_Casia_CenterLoss_2018(InceptionResnet):
         )
 
 
-class InceptionResnetv1_MsCeleb_CenterLoss_2018(InceptionResnet):
+class InceptionResnetv1_MsCeleb_CenterLoss_2018(TransformTensorflow):
     """
     InceptionResnet v1 model trained in 2018 using the MsCeleb dataset in the context of the work:
 
@@ -237,7 +237,7 @@ class InceptionResnetv1_MsCeleb_CenterLoss_2018(InceptionResnet):
         )
 
 
-class FaceNetSanderberg_20170512_110547(InceptionResnet):
+class FaceNetSanderberg_20170512_110547(TransformTensorflow):
     """
     Wrapper for the free FaceNet from David Sanderberg model 20170512_110547:
     https://github.com/davidsandberg/facenet

bob/bio/face/tensorflow/preprocessing.py

@@ -42,9 +42,10 @@ def get_preprocessor(output_shape):
             layers.experimental.preprocessing.RandomFlip("horizontal"),
             # FIXED_STANDARDIZATION from https://github.com/davidsandberg/facenet
             # [-0.99609375, 0.99609375]
-            layers.experimental.preprocessing.Rescaling(
-                scale=1 / 128, offset=-127.5 / 128
-            ),
+            # layers.experimental.preprocessing.Rescaling(
+            #    scale=1 / 128, offset=-127.5 / 128
+            # ),
+            layers.experimental.preprocessing.Rescaling(scale=1 / 255, offset=0),
         ]
     )
     return preprocessor
@@ -100,7 +101,9 @@ def prepare_dataset(
         ignore_order = tf.data.Options()
         ignore_order.experimental_deterministic = False
         ds = ds.with_options(ignore_order)
-    ds = ds.map(partial(decode_tfrecords, data_shape=data_shape)).prefetch(buffer_size=autotune)
+    ds = ds.map(partial(decode_tfrecords, data_shape=data_shape)).prefetch(
+        buffer_size=autotune
+    )
     if shuffle:
         ds = ds.shuffle(shuffle_buffer).repeat(epochs)
     preprocessor = get_preprocessor(output_shape)

cnn_training/arcface.py

@@ -52,11 +52,12 @@ validation-tf-record-path: "/path/lfw_pairs.tfrecord"
 
 
 Usage:
-    arcface.py <config-yaml> <checkpoint_path> 
+    arcface.py <config-yaml> <checkpoint_path> [--pre-train]
     arcface.py -h | --help
 
 Options:
   -h --help             Show this screen.
+  --pre-train           If set pretrains the CNN with the crossentropy softmax for 2 epochs  
   arcface.py arcface -h | help
 
 """
@@ -67,6 +68,7 @@ from functools import partial
 import pkg_resources
 import tensorflow as tf
 from bob.learn.tensorflow.models.inception_resnet_v2 import InceptionResNetV2
+from bob.learn.tensorflow.models import resnet50v1
 from bob.learn.tensorflow.metrics import predict_using_tensors
 from tensorflow.keras import layers
 from bob.learn.tensorflow.callbacks import add_backup_callback
@@ -99,6 +101,9 @@ BACKBONES["inception-resnet-v2"] = InceptionResNetV2
 BACKBONES["efficientnet-B0"] = tf.keras.applications.EfficientNetB0
 BACKBONES["resnet50"] = tf.keras.applications.ResNet50
 BACKBONES["mobilenet-v2"] = tf.keras.applications.MobileNetV2
+# from bob.learn.tensorflow.models.lenet5 import LeNet5_simplified
+
+BACKBONES["resnet50v1"] = resnet50v1
 
 ##############################
 # SOLVER SPECIFICATIONS
@@ -150,7 +155,7 @@ VALIDATION_BATCH_SIZE = 38
 
 
 def create_model(
-    n_classes, model_spec, backbone, bottleneck, dropout_rate, input_shape
+    n_classes, model_spec, backbone, bottleneck, dropout_rate, input_shape, pre_train
 ):
 
     if backbone == "inception-resnet-v2":
@@ -166,27 +171,34 @@ def create_model(
     pre_model = add_bottleneck(
         pre_model, bottleneck_size=bottleneck, dropout_rate=dropout_rate
     )
-    pre_model = add_top(pre_model, n_classes=n_classes)
 
-    float32_layer = layers.Activation("linear", dtype="float32")
+    embeddings = pre_model.get_layer("embeddings").output
+    labels = tf.keras.layers.Input([], name="label")
 
-    embeddings = tf.nn.l2_normalize(
-        pre_model.get_layer("embeddings/BatchNorm").output, axis=1
-    )
+    if pre_train:
+        # pre_model = add_top(pre_model, n_classes=n_classes)
+        logits_premodel = ArcFaceLayer(
+            n_classes,
+            s=model_spec["arcface"]["s"],
+            m=model_spec["arcface"]["m"],
+            arc=False,
+        )(embeddings, None)
 
-    logits_premodel = float32_layer(pre_model.get_layer("logits").output)
+        # Wrapping the embedding validation
+        # logits_premodel = pre_model.get_layer("logits").output
 
-    # Wrapping the embedding validation
-    pre_model = EmbeddingValidation(
-        pre_model.input, outputs=[logits_premodel, embeddings], name=pre_model.name
-    )
+        pre_model = EmbeddingValidation(
+            pre_model.input, outputs=[logits_premodel, embeddings], name=pre_model.name
+        )
 
     ################################
     ## Creating the specific models
     if "arcface" in model_spec:
-        labels = tf.keras.layers.Input([], name="label")
         logits_arcface = ArcFaceLayer(
-            n_classes, s=model_spec["arcface"]["s"], m=model_spec["arcface"]["m"]
+            n_classes,
+            s=model_spec["arcface"]["s"],
+            m=model_spec["arcface"]["m"],
+            arc=True,
         )(embeddings, labels)
         arc_model = ArcFaceModel(
             inputs=(pre_model.input, labels), outputs=[logits_arcface, embeddings]
@@ -221,19 +233,38 @@ def create_model(
 
 
 def build_and_compile_models(
-    n_classes, optimizer, model_spec, backbone, bottleneck, dropout_rate, input_shape
+    n_classes,
+    optimizer,
+    model_spec,
+    backbone,
+    bottleneck,
+    dropout_rate,
+    input_shape,
+    pre_train,
 ):
     pre_model, arc_model = create_model(
-        n_classes, model_spec, backbone, bottleneck, dropout_rate, input_shape
+        n_classes,
+        model_spec,
+        backbone,
+        bottleneck,
+        dropout_rate,
+        input_shape,
+        pre_train,
     )
 
     cross_entropy = tf.keras.losses.SparseCategoricalCrossentropy(
         from_logits=True, name="cross_entropy"
     )
 
-    pre_model.compile(optimizer=optimizer, loss=cross_entropy, metrics=["accuracy"])
+    # Compile the Cross-entropy model if the case
+    if pre_train:
+        pre_model.compile(
+            optimizer=optimizer, loss=cross_entropy, metrics=["accuracy"],
+        )
 
-    arc_model.compile(optimizer=optimizer, loss=cross_entropy, metrics=["accuracy"])
+    arc_model.compile(
+        optimizer=optimizer, loss=cross_entropy, run_eagerly=True, metrics=["accuracy"]
+    )
 
     return pre_model, arc_model
 
@@ -252,6 +283,7 @@ def train_and_evaluate(
     face_size,
     validation_path,
     lerning_rate_schedule,
+    pre_train=False,
 ):
 
     # number of training steps to do before validating a model. This also defines an epoch
@@ -300,6 +332,7 @@ def train_and_evaluate(
         bottleneck=bottleneck,
         dropout_rate=dropout_rate,
         input_shape=OUTPUT_SHAPE + (3,),
+        pre_train=pre_train,
     )
 
     def scheduler(epoch, lr):
@@ -312,8 +345,10 @@ def train_and_evaluate(
 
         if epoch in range(200):
             return 1 * lr
+        elif epoch < 1000:
+            return lr * np.exp(-0.005)
         else:
-            return lr * tf.math.exp(-0.01)
+            return 0.0001
 
     if lerning_rate_schedule == "cosine-decay-restarts":
         decay_steps = 50
@@ -339,16 +374,22 @@ def train_and_evaluate(
     }
 
     callbacks = add_backup_callback(callbacks, backup_dir=f"{checkpoint_path}/backup")
-    # STEPS_PER_EPOCH
-    pre_model.fit(
-        train_ds,
-        epochs=2,
-        validation_data=val_ds,
-        steps_per_epoch=STEPS_PER_EPOCH,
-        validation_steps=VALIDATION_SAMPLES // VALIDATION_BATCH_SIZE,
-        callbacks=callbacks,
-        verbose=2,
-    )
+
+    # Train the Cross-entropy model if the case
+
+    if pre_train:
+        # STEPS_PER_EPOCH
+        pre_model.fit(
+            train_ds,
+            epochs=20,
+            validation_data=val_ds,
+            steps_per_epoch=STEPS_PER_EPOCH,
+            validation_steps=VALIDATION_SAMPLES // VALIDATION_BATCH_SIZE,
+            callbacks=callbacks,
+            verbose=2,
+        )
+        # Copying the last variable
+        arc_model.trainable_variables[-1].assign(pre_model.trainable_variables[-1])
 
     # STEPS_PER_EPOCH
     # epochs=epochs * KERAS_EPOCH_MULTIPLIER,
@@ -404,6 +445,9 @@ if __name__ == "__main__":
         dropout_rate=float(config["dropout-rate"]),
         face_size=int(config["face-size"]),
         validation_path=config["validation-tf-record-path"],
-        lerning_rate_schedule=config["lerning-rate-schedule"],
+        lerning_rate_schedule=config["lerning-rate-schedule"]
+        if "lerning-rate-schedule" in config
+        else None,
+        pre_train=args["--pre-train"],
     )
 

setup.py

@@ -146,6 +146,8 @@ setup(
             "lgbphs = bob.bio.face.config.baseline.lgbphs:pipeline",
             "lda = bob.bio.face.config.baseline.lda:pipeline",
             "dummy = bob.bio.face.config.baseline.dummy:pipeline",
+            "resnet50-msceleb-arcface-2021 = bob.bio.face.config.baseline.resnet50_msceleb_arcface_2021:pipeline",
+            "mobilenetv2-msceleb-arcface-2021 = bob.bio.face.config.baseline.mobilenetv2_msceleb_arcface_2021",
         ],
         "bob.bio.config": [
             "facenet-sanderberg = bob.bio.face.config.baseline.facenet_sanderberg",
@@ -174,6 +176,8 @@ setup(
             "fargo  = bob.bio.face.config.database.fargo",
             "meds = bob.bio.face.config.database.meds",
             "morph = bob.bio.face.config.database.morph",
+            "resnet50-msceleb-arcface-2021 = bob.bio.face.config.baseline.resnet50_msceleb_arcface_2021",
+            "mobilenetv2-msceleb-arcface-2021 = bob.bio.face.config.baseline.mobilenetv2_msceleb_arcface_2021",
         ],
         "bob.bio.cli": [
             "display-face-annotations          = bob.bio.face.script.display_face_annotations:display_face_annotations",