Commit fda2c82d authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Reestructured MXNet

parent 0c5c4e4f
from bob.bio.face.embeddings.mxnet_models import ArcFaceInsightFace
from bob.bio.face.utils import lookup_config_from_database
from bob.bio.face.config.baseline.templates import arcface_baseline
from bob.bio.face.embeddings.mxnet import ArcFaceInsightFace_LResNet100
from bob.bio.face.config.baseline.helpers import embedding_transformer_112x112
from bob.bio.base.pipelines.vanilla_biometrics import (
Distance,
VanillaBiometricsPipeline,
)
if "database" in locals():
annotation_type = database.annotation_type
fixed_positions = database.fixed_positions
memory_demanding = (
database.memory_demanding if hasattr(database, "memory_demanding") else False
)
else:
annotation_type = None
fixed_positions = None
memory_demanding = False
annotation_type, fixed_positions, memory_demanding = lookup_config_from_database(
locals().get("database")
)
def load(annotation_type, fixed_positions=None):
transformer = embedding_transformer_112x112(
ArcFaceInsightFace_LResNet100(memory_demanding=memory_demanding),
annotation_type,
fixed_positions,
color_channel="rgb",
)
return arcface_baseline(
embedding=ArcFaceInsightFace(memory_demanding=memory_demanding),
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Yu Linghu & Xinyi Zhang <yu.linghu@uzh.ch, xinyi.zhang@uzh.ch>
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.utils import check_array
from bob.extension.download import get_file
import numpy as np
import os
class MxNetTransformer(TransformerMixin, BaseEstimator):
"""
Base Transformer for MxNet architectures.
Parameters:
-----------
checkpoint_path : str
Path containing the checkpoint
config : str
json file containing the DNN spec
use_gpu: bool
"""
def __init__(
self,
checkpoint_path=None,
config=None,
use_gpu=False,
memory_demanding=False,
**kwargs,
):
super().__init__(**kwargs)
self.checkpoint_path = checkpoint_path
self.config = config
self.use_gpu = use_gpu
self.model = None
self.memory_demanding = memory_demanding
def _load_model(self):
import mxnet as mx
from mxnet import gluon
import warnings
ctx = mx.gpu() if self.use_gpu else mx.cpu()
with warnings.catch_warnings():
warnings.simplefilter("ignore")
deserialized_net = gluon.nn.SymbolBlock.imports(
self.config, ["data"], self.checkpoint_path, ctx=ctx
)
self.model = deserialized_net
def transform(self, X):
import mxnet as mx
if self.model is None:
self._load_model()
X = check_array(X, allow_nd=True)
def _transform(X):
X = mx.nd.array(X)
db = mx.io.DataBatch(data=(X,))
self.model.forward(db, is_train=False)
return self.model.get_outputs()[0].asnumpy()
if self.memory_demanding:
return np.array([_transform(x[None, ...]) for x in X])
else:
return _transform(X)
def __getstate__(self):
# Handling unpicklable objects
d = self.__dict__.copy()
d["model"] = None
return d
def _more_tags(self):
return {"stateless": True, "requires_fit": False}
class ArcFaceInsightFace_LResNet100(MxNetTransformer):
"""
Extracts features using deep face recognition models under MxNet Interfaces.
Users can download the pretrained face recognition models with MxNet Interface. The path to downloaded models (and weights) should be specified while calling this class, usually in the configuration file of an experiment.
Examples: (Pretrained ResNet models): `LResNet100E-IR,ArcFace@ms1m-refine-v2 <https://github.com/deepinsight/insightface>`_
The extracted features can be combined with different the algorithms.
"""
def __init__(self, memory_demanding=False, use_gpu=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/mxnet/arcface_r100_v1_mxnet.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/mxnet/arcface_r100_v1_mxnet.tar.gz",
]
filename = get_file(
"arcface_r100_v1_mxnet.tar.gz",
urls,
cache_subdir="data/mxnet/arcface_r100_v1_mxnet",
file_hash="050ce7d6e731e560127c705f61391f48",
extract=True,
)
path = os.path.dirname(filename)
checkpoint_path = os.path.join(path, "model-symbol.json")
config = os.path.join(path, "model-0000.params")
super(ArcFaceInsightFace_LResNet100, self).__init__(
checkpoint_path=checkpoint_path,
config=config,
use_gpu=use_gpu,
memory_demanding=memory_demanding,
)
def _load_model(self):
import mxnet as mx
sym, arg_params, aux_params = mx.model.load_checkpoint(
os.path.join(os.path.dirname(self.checkpoint_path), "model"), 0
)
all_layers = sym.get_internals()
sym = all_layers["fc1_output"]
# LOADING CHECKPOINT
ctx = mx.gpu() if self.use_gpu else mx.cpu()
model = mx.mod.Module(symbol=sym, context=ctx, label_names=None)
data_shape = (1, 3, 112, 112)
model.bind(data_shapes=[("data", data_shape)])
model.set_params(arg_params, aux_params)
self.model = model
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# Tranformers based on tensorflow
import os
import pkg_resources
from bob.learn.tensorflow.utils.image import to_channels_last
from sklearn.base import TransformerMixin, BaseEstimator
from bob.extension.download import get_file
from sklearn.utils import check_array
import numpy as np
import tensorflow as tf
def sanderberg_rescaling():
# FIXED_STANDARDIZATION from https://github.com/davidsandberg/facenet
# [-0.99609375, 0.99609375]
preprocessor = preprocessing.Rescaling(scale=1 / 128, offset=-127.5 / 128)
return preprocessor
class TensorflowTransformer(TransformerMixin, BaseEstimator):
"""
Base Transformer for Tensorflow architectures.
Parameters
----------
checkpoint_path: str
Path containing the checkpoint
preprocessor:
Preprocessor function
memory_demanding bool
If `True`, the `transform` method will run one sample at the time.
This is useful when there is not enough memory available to forward big chucks of data.
"""
def __init__(
self, checkpoint_path, preprocessor=None, memory_demanding=False, **kwargs
):
super().__init__(**kwargs)
self.checkpoint_path = checkpoint_path
self.model = None
self.preprocessor = preprocessor
self.memory_demanding = memory_demanding
def load_model(self):
self.model = tf.keras.models.load_model(self.checkpoint_path)
def transform(self, X):
def _transform(X):
X = tf.convert_to_tensor(X)
X = to_channels_last(X)
if X.shape[-3:] != self.model.input_shape[-3:]:
raise ValueError(
f"Image shape {X.shape} not supported. Expected {self.model.input_shape}"
)
return self.inference(X).numpy()
if self.model is None:
self.load_model()
X = check_array(X, allow_nd=True)
if self.memory_demanding:
return np.array([_transform(x[None, ...]) for x in X])
else:
return _transform(X)
def __getstate__(self):
# Handling unpicklable objects
d = self.__dict__.copy()
d["model"] = None
return d
def inference(self, X):
if self.preprocessor is not None:
X = self.preprocessor(tf.cast(X, "float32"))
prelogits = self.model.predict_on_batch(X)
embeddings = tf.math.l2_normalize(prelogits, axis=-1)
return embeddings
def _more_tags(self):
return {"stateless": True, "requires_fit": False}
def __del__(self):
self.model = None
class InceptionResnetv2_MsCeleb_CenterLoss_2018(TensorflowTransformer):
"""
InceptionResnet v2 model trained in 2018 using the MSCeleb dataset in the context of the work:
Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
]
filename = get_file(
"inceptionresnetv2_msceleb_centerloss_2018.tar.gz",
urls,
cache_subdir="data/tensorflow/inceptionresnetv2_msceleb_centerloss_2018",
file_hash="7c0aa46bba16c01768a38594a3b4c14d",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(InceptionResnetv2_MsCeleb_CenterLoss_2018, self).__init__(
checkpoint_path,
preprocessor=tf.image.per_image_standardization,
memory_demanding=memory_demanding,
)
class InceptionResnetv2_Casia_CenterLoss_2018(TensorflowTransformer):
"""
InceptionResnet v2 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz",
]
filename = get_file(
"inceptionresnetv2_casia_centerloss_2018.tar.gz",
urls,
cache_subdir="data/tensorflow/inceptionresnetv2_casia_centerloss_2018",
file_hash="1e0b62e45430a8d7516d7a6101a24c40",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(InceptionResnetv2_Casia_CenterLoss_2018, self).__init__(
checkpoint_path,
preprocessor=tf.image.per_image_standardization,
memory_demanding=memory_demanding,
)
class InceptionResnetv1_Casia_CenterLoss_2018(TensorflowTransformer):
"""
InceptionResnet v1 model trained in 2018 using the CasiaWebFace dataset in the context of the work:
Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz",
]
filename = get_file(
"inceptionresnetv1_casia_centerloss_2018.tar.gz",
urls,
cache_subdir="data/tensorflow/inceptionresnetv1_casia_centerloss_2018",
file_hash="6601e6f6840ae863c7daf31a7c6b9a27",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(InceptionResnetv1_Casia_CenterLoss_2018, self).__init__(
checkpoint_path,
preprocessor=tf.image.per_image_standardization,
memory_demanding=memory_demanding,
)
class InceptionResnetv1_MsCeleb_CenterLoss_2018(TensorflowTransformer):
"""
InceptionResnet v1 model trained in 2018 using the MsCeleb dataset in the context of the work:
Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816.
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
]
filename = get_file(
"inceptionresnetv1_msceleb_centerloss_2018.tar.gz",
urls,
cache_subdir="data/tensorflow/inceptionresnetv1_msceleb_centerloss_2018",
file_hash="1ca0149619e4e9320a927ea65b2b5521",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(InceptionResnetv1_MsCeleb_CenterLoss_2018, self).__init__(
checkpoint_path,
preprocessor=tf.image.per_image_standardization,
memory_demanding=memory_demanding,
)
class FaceNetSanderberg_20170512_110547(TensorflowTransformer):
"""
Wrapper for the free FaceNet from David Sanderberg model 20170512_110547:
https://github.com/davidsandberg/facenet
And for a preprocessor you can use::
from bob.bio.face.preprocessor import FaceCrop
# This is the size of the image that this model expects
CROPPED_IMAGE_HEIGHT = 160
CROPPED_IMAGE_WIDTH = 160
# eye positions for frontal images
RIGHT_EYE_POS = (46, 53)
LEFT_EYE_POS = (46, 107)
# Crops the face using eye annotations
preprocessor = FaceCrop(
cropped_image_size=(CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH),
cropped_positions={'leye': LEFT_EYE_POS, 'reye': RIGHT_EYE_POS},
color_channel='rgb'
)
"""
def __init__(self, memory_demanding=False):
urls = [
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/facenet_sanderberg_20170512_110547.tar.gz"
]
filename = get_file(
"facenet_sanderberg_20170512_110547.tar.gz",
urls,
cache_subdir="data/tensorflow/facenet_sanderberg_20170512_110547",
file_hash="734d1c997c10acdcdffc79fb51a2e715",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(FaceNetSanderberg_20170512_110547, self).__init__(
checkpoint_path,
tf.image.per_image_standardization,
memory_demanding=memory_demanding,
)
class Resnet50_MsCeleb_ArcFace_2021(TensorflowTransformer):
"""
Resnet50 Backbone trained with the MSCeleb 1M database.
The bottleneck layer (a.k.a embedding) has 512d.
The configuration file used to trained is:
.. warning::
This configuration file might change in future releases
```yaml
batch-size: 128
face-size: 112
face-output_size: 112
n-classes: 85742
## Backbone
backbone: 'resnet50'
head: 'arcface'
s: 10
bottleneck: 512
m: 0.5
# Training parameters
solver: "sgd"
lr: 0.1
dropout-rate: 0.5
epochs: 500
train-tf-record-path: "<PATH>"
validation-tf-record-path: "<PATH>"
```
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_msceleb_arcface_2021.tar.gz",
]
filename = get_file(
"resnet50_msceleb_arcface_2021.tar.gz",
urls,
cache_subdir="data/tensorflow/resnet50_msceleb_arcface_2021",
file_hash="1e4b9791669ef79cf8ed80a6fc830205",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(Resnet50_MsCeleb_ArcFace_2021, self).__init__(
checkpoint_path,
preprocessor=lambda X: X / 255.0,
memory_demanding=memory_demanding,
)
def inference(self, X):
if self.preprocessor is not None:
X = self.preprocessor(tf.cast(X, "float32"))
prelogits = self.model.predict_on_batch(X)[0]
embeddings = tf.math.l2_normalize(prelogits, axis=-1)
return embeddings
class Resnet50_VGG2_ArcFace_2021(TensorflowTransformer):
"""
Resnet50 Backbone trained with the VGG2 database.
The bottleneck layer (a.k.a embedding) has 512d.
The configuration file used to trained is:
.. warning::
This configuration file might change in future releases
```yaml
batch-size: 128
face-size: 112
face-output_size: 112
n-classes: 8631
## Backbone
backbone: 'resnet50'
head: 'arcface'
s: 64
bottleneck: 512
m: 0.5
# Training parameters
solver: "sgd"
lr: 0.1
dropout-rate: 0.5
epochs: 1047
train-tf-record-path: "<PATH>"
validation-tf-record-path: "<PATH>"
```
"""
def __init__(self, memory_demanding=False):
urls = [
"https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz",
"http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz",
]
filename = get_file(
"resnet50_vgg2_arcface_2021.tar.gz",
urls,
cache_subdir="data/tensorflow/resnet50_vgg2_arcface_2021",
file_hash="64f89c8cb55e7a0d9c7e13ff412b6a13",
extract=True,
)
checkpoint_path = os.path.dirname(filename)
super(Resnet50_VGG2_ArcFace_2021, self).__init__(
checkpoint_path,
preprocessor=lambda X: X / 255.0,
memory_demanding=memory_demanding,
)
def inference(self, X):
if self.preprocessor is not None:
X = self.preprocessor(tf.cast(X, "float32"))
prelogits = self.model.predict_on_batch(X)
embeddings = tf.math.l2_normalize(prelogits, axis=-1)
return embeddings
class MobileNetv2_MsCeleb_ArcFace_2021(TensorflowTransformer):
"""
MobileNet Backbone trained with the MSCeleb 1M database.
The bottleneck layer (a.k.a embedding) has 512d.
The configuration file used to trained is:
.. warning::
This configuration file might change in future releases
```yaml
batch-size: 128
face-size: 112
face-output_size: 112
n-classes: 85742
## Backbone
backbone: 'mobilenet-v2'
head: 'arcface'
s: 10
bottleneck: 512
m: 0.5
# Training parameters
solver: "sgd"
lr: 0.01
dropout-rate: 0.5
epochs: 500
train-tf-record-path: "<PATH>"