Reestructured MXNet

bob/bio/face/config/baseline/arcface_insightface.py

-from bob.bio.face.embeddings.mxnet_models import ArcFaceInsightFace
-from bob.bio.face.utils import lookup_config_from_database
-from bob.bio.face.config.baseline.templates import arcface_baseline
+from bob.bio.face.embeddings.mxnet import ArcFaceInsightFace_LResNet100
+from bob.bio.face.config.baseline.helpers import embedding_transformer_112x112
+from bob.bio.base.pipelines.vanilla_biometrics import (
+    Distance,
+    VanillaBiometricsPipeline,
+)
+
+
+if "database" in locals():
+    annotation_type = database.annotation_type
+    fixed_positions = database.fixed_positions
+    memory_demanding = (
+        database.memory_demanding if hasattr(database, "memory_demanding") else False
+    )
+else:
+    annotation_type = None
+    fixed_positions = None
+    memory_demanding = False
 
 annotation_type, fixed_positions, memory_demanding = lookup_config_from_database(
     locals().get("database")
 )
 
+
 def load(annotation_type, fixed_positions=None):
+    transformer = embedding_transformer_112x112(
+        ArcFaceInsightFace_LResNet100(memory_demanding=memory_demanding),
+        annotation_type,
+        fixed_positions,
+        color_channel="rgb",
+    )
 
     return arcface_baseline(
         embedding=ArcFaceInsightFace(memory_demanding=memory_demanding),

bob/bio/face/embeddings/mxnet.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Yu Linghu & Xinyi Zhang <yu.linghu@uzh.ch, xinyi.zhang@uzh.ch>
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+
+from sklearn.base import TransformerMixin, BaseEstimator
+from sklearn.utils import check_array
+from bob.extension.download import get_file
+import numpy as np
+import os
+
+
+class MxNetTransformer(TransformerMixin, BaseEstimator):
+
+    """
+    Base Transformer for MxNet architectures.
+
+    Parameters:
+    -----------
+
+      checkpoint_path : str
+         Path containing the checkpoint
+
+      config : str
+         json file containing the DNN spec
+
+      use_gpu: bool
+    """
+
+    def __init__(
+        self,
+        checkpoint_path=None,
+        config=None,
+        use_gpu=False,
+        memory_demanding=False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.checkpoint_path = checkpoint_path
+        self.config = config
+        self.use_gpu = use_gpu
+        self.model = None
+        self.memory_demanding = memory_demanding
+
+    def _load_model(self):
+        import mxnet as mx
+        from mxnet import gluon
+        import warnings
+
+        ctx = mx.gpu() if self.use_gpu else mx.cpu()
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            deserialized_net = gluon.nn.SymbolBlock.imports(
+                self.config, ["data"], self.checkpoint_path, ctx=ctx
+            )
+
+        self.model = deserialized_net
+
+    def transform(self, X):
+
+        import mxnet as mx
+
+        if self.model is None:
+            self._load_model()
+
+        X = check_array(X, allow_nd=True)
+
+        def _transform(X):
+            X = mx.nd.array(X)
+            db = mx.io.DataBatch(data=(X,))
+            self.model.forward(db, is_train=False)
+            return self.model.get_outputs()[0].asnumpy()
+
+        if self.memory_demanding:
+            return np.array([_transform(x[None, ...]) for x in X])
+        else:
+            return _transform(X)
+
+    def __getstate__(self):
+        # Handling unpicklable objects
+
+        d = self.__dict__.copy()
+        d["model"] = None
+        return d
+
+    def _more_tags(self):
+        return {"stateless": True, "requires_fit": False}
+
+
+class ArcFaceInsightFace_LResNet100(MxNetTransformer):
+    """
+    Extracts features using deep face recognition models under MxNet Interfaces.
+  
+    Users can download the pretrained face recognition models with MxNet Interface. The path to downloaded models (and weights) should be specified while calling this class, usually in the configuration file of an experiment. 
+ 
+    Examples: (Pretrained ResNet models): `LResNet100E-IR,ArcFace@ms1m-refine-v2 <https://github.com/deepinsight/insightface>`_  
+  
+    The extracted features can be combined with different the algorithms.  
+
+    """
+
+    def __init__(self, memory_demanding=False, use_gpu=False):
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/mxnet/arcface_r100_v1_mxnet.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/mxnet/arcface_r100_v1_mxnet.tar.gz",
+        ]
+        filename = get_file(
+            "arcface_r100_v1_mxnet.tar.gz",
+            urls,
+            cache_subdir="data/mxnet/arcface_r100_v1_mxnet",
+            file_hash="050ce7d6e731e560127c705f61391f48",
+            extract=True,
+        )
+        path = os.path.dirname(filename)
+        checkpoint_path = os.path.join(path, "model-symbol.json")
+        config = os.path.join(path, "model-0000.params")
+
+        super(ArcFaceInsightFace_LResNet100, self).__init__(
+            checkpoint_path=checkpoint_path,
+            config=config,
+            use_gpu=use_gpu,
+            memory_demanding=memory_demanding,
+        )
+
+    def _load_model(self):
+        import mxnet as mx
+
+        sym, arg_params, aux_params = mx.model.load_checkpoint(
+            os.path.join(os.path.dirname(self.checkpoint_path), "model"), 0
+        )
+
+        all_layers = sym.get_internals()
+        sym = all_layers["fc1_output"]
+
+        # LOADING CHECKPOINT
+        ctx = mx.gpu() if self.use_gpu else mx.cpu()
+        model = mx.mod.Module(symbol=sym, context=ctx, label_names=None)
+        data_shape = (1, 3, 112, 112)
+        model.bind(data_shapes=[("data", data_shape)])
+        model.set_params(arg_params, aux_params)
+
+        self.model = model

bob/bio/face/embeddings/tensorflow.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+
+# Tranformers based on tensorflow
+
+
+import os
+import pkg_resources
+from bob.learn.tensorflow.utils.image import to_channels_last
+from sklearn.base import TransformerMixin, BaseEstimator
+from bob.extension.download import get_file
+from sklearn.utils import check_array
+import numpy as np
+import tensorflow as tf
+
+
+def sanderberg_rescaling():
+    # FIXED_STANDARDIZATION from https://github.com/davidsandberg/facenet
+    # [-0.99609375, 0.99609375]
+    preprocessor = preprocessing.Rescaling(scale=1 / 128, offset=-127.5 / 128)
+    return preprocessor
+
+
+class TensorflowTransformer(TransformerMixin, BaseEstimator):
+    """
+    Base Transformer for Tensorflow architectures.
+
+    Parameters
+    ----------
+
+    checkpoint_path: str
+       Path containing the checkpoint
+
+    preprocessor:
+        Preprocessor function
+
+    memory_demanding bool
+        If `True`, the `transform` method will run one sample at the time.
+        This is useful when there is not enough memory available to forward big chucks of data.
+    """
+
+    def __init__(
+        self, checkpoint_path, preprocessor=None, memory_demanding=False, **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.checkpoint_path = checkpoint_path
+        self.model = None
+        self.preprocessor = preprocessor
+        self.memory_demanding = memory_demanding
+
+    def load_model(self):
+        self.model = tf.keras.models.load_model(self.checkpoint_path)
+
+    def transform(self, X):
+        def _transform(X):
+            X = tf.convert_to_tensor(X)
+            X = to_channels_last(X)
+
+            if X.shape[-3:] != self.model.input_shape[-3:]:
+                raise ValueError(
+                    f"Image shape {X.shape} not supported. Expected {self.model.input_shape}"
+                )
+
+            return self.inference(X).numpy()
+
+        if self.model is None:
+            self.load_model()
+
+        X = check_array(X, allow_nd=True)
+
+        if self.memory_demanding:
+            return np.array([_transform(x[None, ...]) for x in X])
+        else:
+            return _transform(X)
+
+    def __getstate__(self):
+        # Handling unpicklable objects
+        d = self.__dict__.copy()
+        d["model"] = None
+        return d
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+
+    def _more_tags(self):
+        return {"stateless": True, "requires_fit": False}
+
+    def __del__(self):
+        self.model = None
+
+
+class InceptionResnetv2_MsCeleb_CenterLoss_2018(TensorflowTransformer):
+    """
+    InceptionResnet v2 model trained in 2018 using the MSCeleb dataset in the context of the work:
+
+    Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
+
+    """
+
+    def __init__(self, memory_demanding=False):
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
+        ]
+
+        filename = get_file(
+            "inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/inceptionresnetv2_msceleb_centerloss_2018",
+            file_hash="7c0aa46bba16c01768a38594a3b4c14d",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(InceptionResnetv2_MsCeleb_CenterLoss_2018, self).__init__(
+            checkpoint_path,
+            preprocessor=tf.image.per_image_standardization,
+            memory_demanding=memory_demanding,
+        )
+
+
+class InceptionResnetv2_Casia_CenterLoss_2018(TensorflowTransformer):
+    """
+    InceptionResnet v2 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
+
+    Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
+
+    """
+
+    def __init__(self, memory_demanding=False):
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz",
+        ]
+
+        filename = get_file(
+            "inceptionresnetv2_casia_centerloss_2018.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/inceptionresnetv2_casia_centerloss_2018",
+            file_hash="1e0b62e45430a8d7516d7a6101a24c40",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(InceptionResnetv2_Casia_CenterLoss_2018, self).__init__(
+            checkpoint_path,
+            preprocessor=tf.image.per_image_standardization,
+            memory_demanding=memory_demanding,
+        )
+
+
+class InceptionResnetv1_Casia_CenterLoss_2018(TensorflowTransformer):
+    """
+    InceptionResnet v1 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
+
+    Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
+
+    """
+
+    def __init__(self, memory_demanding=False):
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz",
+        ]
+
+        filename = get_file(
+            "inceptionresnetv1_casia_centerloss_2018.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/inceptionresnetv1_casia_centerloss_2018",
+            file_hash="6601e6f6840ae863c7daf31a7c6b9a27",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(InceptionResnetv1_Casia_CenterLoss_2018, self).__init__(
+            checkpoint_path,
+            preprocessor=tf.image.per_image_standardization,
+            memory_demanding=memory_demanding,
+        )
+
+
+class InceptionResnetv1_MsCeleb_CenterLoss_2018(TensorflowTransformer):
+    """
+    InceptionResnet v1 model trained in 2018 using the MsCeleb dataset in the context of the work:
+
+    Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
+
+    """
+
+    def __init__(self, memory_demanding=False):
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
+        ]
+
+        filename = get_file(
+            "inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/inceptionresnetv1_msceleb_centerloss_2018",
+            file_hash="1ca0149619e4e9320a927ea65b2b5521",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(InceptionResnetv1_MsCeleb_CenterLoss_2018, self).__init__(
+            checkpoint_path,
+            preprocessor=tf.image.per_image_standardization,
+            memory_demanding=memory_demanding,
+        )
+
+
+class FaceNetSanderberg_20170512_110547(TensorflowTransformer):
+    """
+    Wrapper for the free FaceNet from David Sanderberg model 20170512_110547:
+    https://github.com/davidsandberg/facenet
+
+    And for a preprocessor you can use::
+
+        from bob.bio.face.preprocessor import FaceCrop
+        # This is the size of the image that this model expects
+        CROPPED_IMAGE_HEIGHT = 160
+        CROPPED_IMAGE_WIDTH = 160
+        # eye positions for frontal images
+        RIGHT_EYE_POS = (46, 53)
+        LEFT_EYE_POS = (46, 107)
+        # Crops the face using eye annotations
+        preprocessor = FaceCrop(
+            cropped_image_size=(CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH),
+            cropped_positions={'leye': LEFT_EYE_POS, 'reye': RIGHT_EYE_POS},
+            color_channel='rgb'
+        )
+    """
+
+    def __init__(self, memory_demanding=False):
+        urls = [
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/facenet_sanderberg_20170512_110547.tar.gz"
+        ]
+
+        filename = get_file(
+            "facenet_sanderberg_20170512_110547.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/facenet_sanderberg_20170512_110547",
+            file_hash="734d1c997c10acdcdffc79fb51a2e715",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(FaceNetSanderberg_20170512_110547, self).__init__(
+            checkpoint_path,
+            tf.image.per_image_standardization,
+            memory_demanding=memory_demanding,
+        )
+
+
+class Resnet50_MsCeleb_ArcFace_2021(TensorflowTransformer):
+    """
+    Resnet50 Backbone trained with the MSCeleb 1M database.
+
+    The bottleneck layer (a.k.a embedding) has 512d.
+
+    The configuration file used to trained is:
+
+    .. warning::
+        This configuration file might change in future releases
+
+    ```yaml
+    batch-size: 128
+    face-size: 112
+    face-output_size: 112
+    n-classes: 85742
+
+
+    ## Backbone
+    backbone: 'resnet50'
+    head: 'arcface'
+    s: 10
+    bottleneck: 512
+    m: 0.5
+
+    # Training parameters
+    solver: "sgd"
+    lr: 0.1
+    dropout-rate: 0.5
+    epochs: 500
+
+
+    train-tf-record-path: "<PATH>"
+    validation-tf-record-path: "<PATH>"
+
+    ```
+
+
+    """
+
+    def __init__(self, memory_demanding=False):
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
+        ]
+
+        filename = get_file(
+            "resnet50_msceleb_arcface_2021.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/resnet50_msceleb_arcface_2021",
+            file_hash="1e4b9791669ef79cf8ed80a6fc830205",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(Resnet50_MsCeleb_ArcFace_2021, self).__init__(
+            checkpoint_path,
+            preprocessor=lambda X: X / 255.0,
+            memory_demanding=memory_demanding,
+        )
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)[0]
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+
+
+class Resnet50_VGG2_ArcFace_2021(TensorflowTransformer):
+    """
+    Resnet50 Backbone trained with the VGG2 database.
+
+    The bottleneck layer (a.k.a embedding) has 512d.
+
+    The configuration file used to trained is:
+
+    .. warning::
+        This configuration file might change in future releases
+
+    ```yaml
+    batch-size: 128
+    face-size: 112
+    face-output_size: 112
+    n-classes: 8631
+
+
+    ## Backbone
+    backbone: 'resnet50'
+    head: 'arcface'
+    s: 64
+    bottleneck: 512
+    m: 0.5
+
+    # Training parameters
+    solver: "sgd"
+    lr: 0.1
+    dropout-rate: 0.5
+    epochs: 1047
+
+
+    train-tf-record-path: "<PATH>"
+    validation-tf-record-path: "<PATH>"
+
+    ```
+
+
+    """
+
+    def __init__(self, memory_demanding=False):
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz",
+        ]
+
+        filename = get_file(
+            "resnet50_vgg2_arcface_2021.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/resnet50_vgg2_arcface_2021",
+            file_hash="64f89c8cb55e7a0d9c7e13ff412b6a13",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(Resnet50_VGG2_ArcFace_2021, self).__init__(
+            checkpoint_path,
+            preprocessor=lambda X: X / 255.0,
+            memory_demanding=memory_demanding,
+        )
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+
+
+class MobileNetv2_MsCeleb_ArcFace_2021(TensorflowTransformer):
+    """
+    MobileNet Backbone trained with the MSCeleb 1M database.
+
+    The bottleneck layer (a.k.a embedding) has 512d.
+
+    The configuration file used to trained is:
+
+    .. warning::
+        This configuration file might change in future releases
+
+    ```yaml
+    batch-size: 128
+    face-size: 112
+    face-output_size: 112
+    n-classes: 85742
+
+
+    ## Backbone
+    backbone: 'mobilenet-v2'
+    head: 'arcface'
+    s: 10
+    bottleneck: 512
+    m: 0.5
+
+    # Training parameters
+    solver: "sgd"
+    lr: 0.01
+    dropout-rate: 0.5
+    epochs: 500
+
+
+    train-tf-record-path: "<PATH>"
+    validation-tf-record-path: "<PATH>"
+
+    ```
+
+
+    """
+
+    def __init__(self, memory_demanding=False):
+
+        urls = [
+            "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021.tar.gz",
+            "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021.tar.gz",
+        ]
+
+        filename = get_file(
+            "mobilenet-v2-msceleb-arcface-2021.tar.gz",
+            urls,
+            cache_subdir="data/tensorflow/mobilenet-v2-msceleb-arcface-2021",
+            file_hash="89da9399d36ac89a354b8e8beaed8647",
+            extract=True,
+        )
+        checkpoint_path = os.path.dirname(filename)
+
+        super(MobileNetv2_MsCeleb_ArcFace_2021, self).__init__(
+            checkpoint_path,
+            preprocessor=lambda X: X / 255.0,
+            memory_demanding=memory_demanding,
+        )
+
+    def inference(self, X):
+        if self.preprocessor is not None:
+            X = self.preprocessor(tf.cast(X, "float32"))
+
+        prelogits = self.model.predict_on_batch(X)[0]
+        embeddings = tf.math.l2_normalize(prelogits, axis=-1)
+        return embeddings
+

bob/bio/face/test/test_baselines.py

@@ -160,16 +160,16 @@ def test_opencv_pipe():
     run_baseline("opencv-pipe", target_scores=None)
 
 
-@pytest.mark.slow
-@is_library_available("mxnet")
-def test_mxnet_pipe():
-    run_baseline("mxnet-pipe", target_scores=None)
+# @pytest.mark.slow
+# @is_library_available("mxnet")
+# def test_mxnet_pipe():
+# run_baseline("mxnet-pipe", target_scores=None)
 
 
-@pytest.mark.slow
-@is_library_available("tensorflow")
-def test_tf_pipe():
-    run_baseline("tf-pipe", target_scores=None)
+# @pytest.mark.slow
+# @is_library_available("tensorflow")
+# def test_tf_pipe():
+#    run_baseline("tf-pipe", target_scores=None)
 
 
 @pytest.mark.slow