2

bob/bio/face/annotator/__init__.py

 import bob.ip.facedetect
 
+
 def bounding_box_to_annotations(bbx):
     """Converts :any:`bob.ip.facedetect.BoundingBox` to dictionary annotations.
 
@@ -87,4 +88,4 @@ __appropriate__(
     BobIpTinyface,
 )
 
-__all__ = [_ for _ in dir() if not _.startswith('_')]
+__all__ = [_ for _ in dir() if not _.startswith('_')]
\ No newline at end of file

bob/bio/face/annotator/bobiptinyface.py

@@ -26,10 +26,7 @@ class BobIpTinyface(Base):
             Annotations with (topleft, bottomright) keys (or None).
         """
 
-        annotations = self.tinyface.detect(image)
-
         if annotations is not None:
-            r = annotations[0]
-            return {"topleft": (r[0], r[1]), "bottomright": (r[2], r[3])}
+            return annotations[0]
         else:
             return None

bob/bio/face/config/annotator/tinyface.py

 from bob.bio.face.annotator import BobIpTinyface
 
-annotator = BobIpTinyface()
+annotator = BobIpTinyface()
\ No newline at end of file

bob/bio/face/config/baseline/mxnet_pipe.py

 import bob.bio.base
 from bob.bio.face.preprocessor import FaceCrop
-from bob.bio.face.extractor import mxnet_model
+from bob.bio.face.extractor import MxNetModel
 from bob.bio.base.algorithm import Distance
 from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
 import scipy.spatial
@@ -39,7 +39,7 @@ transform_extra_arguments = (
 )
 
 
-extractor_transformer = mxnet_model()
+extractor_transformer = MxNetModel()
 
 algorithm = Distance(
     distance_function=scipy.spatial.distance.cosine, is_distance_function=True

bob/bio/face/config/baseline/mxnet_tinyface_pipe.py

+import bob.bio.base
+from bob.bio.face.preprocessor import FaceCrop
+from bob.bio.face.annotator import BobIpTinyface
+from bob.bio.face.extractor import MxNetModel
+
+from bob.bio.base.algorithm import Distance
+from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
+import scipy.spatial
+from bob.bio.base.pipelines.vanilla_biometrics import Distance
+from sklearn.pipeline import make_pipeline
+from bob.pipelines import wrap
+from bob.bio.base.pipelines.vanilla_biometrics import VanillaBiometricsPipeline
+
+
+annotator_transformer = BobIpTinyface()
+
+preprocessor_transformer = FaceCrop(cropped_image_size=(112,112), cropped_positions={'leye':(49,72), 'reye':(49,38)}, color_channel='rgb',annotator=annotator_transformer)
+
+extractor_transformer = MxNetModel()
+
+
+algorithm = Distance(distance_function = scipy.spatial.distance.cosine,is_distance_function = True)
+
+transformer = make_pipeline(
+    wrap(["sample"], preprocessor_transformer),
+    wrap(["sample"], extractor_transformer)
+)
+
+pipeline = VanillaBiometricsPipeline(transformer, algorithm)
+transformer = pipeline.transformer
\ No newline at end of file

bob/bio/face/config/baseline/opencv_pipe.py

@@ -35,6 +35,15 @@ preprocessor_transformer = FaceCrop(
     fixed_positions=fixed_positions,
 )
 
+cropped_positions = {"leye": (100, 140), "reye": (100, 95)}
+# Preprocessor
+preprocessor_transformer = FaceCrop(
+    cropped_image_size=(224, 224),
+    cropped_positions={"leye": (100, 140), "reye": (100, 95)},
+    color_channel="rgb",
+    fixed_positions=fixed_positions,
+)
+
 transform_extra_arguments = (
     None
     if (cropped_positions is None or fixed_positions is not None)

bob/bio/face/config/baseline/opencv_tinyface_pipe.py

+import bob.bio.base
+from bob.bio.face.preprocessor import FaceCrop
+from bob.bio.face.annotator.bobiptinyface import BobIpTinyface
+from bob.bio.base.transformers.preprocessor import PreprocessorTransformer
+
+from bob.bio.face.extractor import opencv_model
+
+from bob.bio.base.algorithm import Distance
+from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
+import scipy.spatial
+from bob.bio.base.pipelines.vanilla_biometrics import Distance
+
+
+from sklearn.pipeline import make_pipeline
+from bob.pipelines import wrap
+from bob.bio.base.pipelines.vanilla_biometrics import VanillaBiometricsPipeline
+
+
+#memory_demanding = False
+#if "database" in locals():
+#    annotation_type = database.annotation_type
+#    fixed_positions = database.fixed_positions
+#    memory_demanding = (
+#        database.memory_demanding if hasattr(database, "memory_demanding") else False
+#    )
+
+#else:
+#    annotation_type = None
+#    fixed_positions = None
+
+
+annotator_transformer = BobIpTinyface()
+
+
+#right_eye=annotator_transformer.annotate()["reye"]
+#left_eye=annotator_transformer.annotate()["leye"]
+#cropped_positions={'leye':left_eye, 'reye':right_eye}
+#topleft = annotator_transformer.annotate()["topleft"]
+#bottomright = annotator_transformer.annotate()["bottomright"]
+cropped_positions={'leye':(49,72), 'reye':(49,38)}
+preprocessor_transformer = FaceCrop(cropped_image_size=(224,224),cropped_positions={'leye':(49,72), 'reye':(49,38)}, color_channel='rgb', annotator = annotator_transformer)
+
+
+
+
+
+
+extractor_transformer = opencv_model()
+
+
+
+
+algorithm = Distance(distance_function = scipy.spatial.distance.cosine,is_distance_function = True)
+
+
+# Chain the Transformers together
+#transformer = make_pipeline(
+#    wrap(["sample"], preprocessor_transformer,transform_extra_arguments=transform_extra_arguments),
+#    wrap(["sample"], extractor_transformer)
+#    # Add more transformers here if needed
+#)
+
+transformer = make_pipeline(
+    wrap(["sample"], preprocessor_transformer),
+    wrap(["sample"], extractor_transformer)
+)
+
+# Assemble the Vanilla Biometric pipeline and execute
+pipeline = VanillaBiometricsPipeline(transformer, algorithm)
+transformer = pipeline.transformer

bob/bio/face/config/baseline/pytorch_pipe_v1.py

 import bob.bio.base
 from bob.bio.face.preprocessor import FaceCrop
-from bob.bio.face.extractor import pytorch_loaded_model
+from bob.bio.face.extractor import PyTorchLoadedModel
 from bob.bio.base.algorithm import Distance
 from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
 import scipy.spatial
@@ -25,9 +25,11 @@ else:
 
 cropped_positions = {"leye": (49, 72), "reye": (49, 38)}
 
+cropped_positions = {"leye": (110, 144), "reye": (110, 96)}
+
 preprocessor_transformer = FaceCrop(
     cropped_image_size=(224, 224),
-    cropped_positions={"leye": (49, 72), "reye": (49, 38)},
+    cropped_positions={"leye": (110, 144), "reye": (110, 96)},
     color_channel="rgb",
     fixed_positions=fixed_positions,
 )
@@ -38,8 +40,14 @@ transform_extra_arguments = (
     else (("annotations", "annotations"),)
 )
 
+transform_extra_arguments = (
+    None
+    if (cropped_positions is None or fixed_positions is not None)
+    else (("annotations", "annotations"),)
+)
+
 
-extractor_transformer = pytorch_loaded_model()
+extractor_transformer = PyTorchLoadedModel()
 
 algorithm = Distance(
     distance_function=scipy.spatial.distance.cosine, is_distance_function=True

bob/bio/face/config/baseline/pytorch_pipe_v2.py

 import bob.bio.base
 from bob.bio.face.preprocessor import FaceCrop
-from bob.bio.face.extractor import pytorch_library_model
+from bob.bio.face.extractor import PyTorchLibraryModel
 from facenet_pytorch import InceptionResnetV1
 from bob.bio.base.algorithm import Distance
 from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
@@ -26,9 +26,11 @@ else:
 
 cropped_positions = {"leye": (49, 72), "reye": (49, 38)}
 
+cropped_positions = {"leye": (110, 144), "reye": (110, 96)}
+
 preprocessor_transformer = FaceCrop(
     cropped_image_size=(224, 224),
-    cropped_positions={"leye": (49, 72), "reye": (49, 38)},
+    cropped_positions={"leye": (110, 144), "reye": (110, 96)},
     color_channel="rgb",
     fixed_positions=fixed_positions,
 )
@@ -40,8 +42,15 @@ transform_extra_arguments = (
 )
 
 
+transform_extra_arguments = (
+    None
+    if (cropped_positions is None or fixed_positions is not None)
+    else (("annotations", "annotations"),)
+)
+
+
 model = InceptionResnetV1(pretrained="vggface2").eval()
-extractor_transformer = pytorch_library_model(model=model)
+extractor_transformer = PyTorchLibraryModel(model=model)
 
 
 algorithm = Distance(

bob/bio/face/config/baseline/tf_pipe.py

 import bob.bio.base
 from bob.bio.face.preprocessor import FaceCrop
-from bob.bio.face.extractor import tf_model
+from bob.bio.face.extractor import TensorFlowModel
 from bob.bio.base.algorithm import Distance
 from bob.bio.base.pipelines.vanilla_biometrics.legacy import BioAlgorithmLegacy
 import scipy.spatial
@@ -24,11 +24,11 @@ else:
 
 
 # Preprocessor
-cropped_positions = {"leye": (49, 72), "reye": (49, 38)}
+cropped_positions = {"leye": (80, 100), "reye": (80, 60)}
 
 preprocessor_transformer = FaceCrop(
     cropped_image_size=(160, 160),
-    cropped_positions={"leye": (49, 72), "reye": (49, 38)},
+    cropped_positions={"leye": (80, 100), "reye": (80, 60)},
     color_channel="rgb",
     fixed_positions=fixed_positions,
 )
@@ -41,7 +41,7 @@ transform_extra_arguments = (
 
 
 # Extractor
-extractor_transformer = tf_model()
+extractor_transformer = TensorFlowModel()
 
 # Algorithm
 algorithm = Distance(

bob/bio/face/extractor/mxnet_resnet.py → bob/bio/face/extractor/MxNetModel.py

@@ -17,7 +17,7 @@ mxnet_resnet_directory = rc["bob.extractor_model.mxnet"]
 mxnet_weight_directory = rc["bob.extractor_weights.mxnet"]
 
 
-class mxnet_model(TransformerMixin, BaseEstimator):
+class MxNetModel(TransformerMixin, BaseEstimator):
 
     """Extracts features using deep face recognition models under MxNet Interfaces.
   

bob/bio/face/extractor/opencv_caffe.py → bob/bio/face/extractor/OpenCVModel.py

@@ -25,7 +25,7 @@ opencv_model_directory = rc["bob.extractor_model.opencv"]
 opencv_model_prototxt = rc["bob.extractor_weights.opencv"]
 
 
-class opencv_model(TransformerMixin, BaseEstimator):
+class OpenCVModel(TransformerMixin, BaseEstimator):
     """Extracts features using deep face recognition models under OpenCV Interface
 
   Users can download the pretrained face recognition models with OpenCV Interface. The path to downloaded models should be specified before running the extractor (usually before running the pipeline file that includes the extractor). That is, set config of the model frame to :py:class:`bob.extractor_model.opencv`, and set config of the parameters to :py:class:`bob.extractor_weights.opencv`. 
@@ -39,8 +39,6 @@ class opencv_model(TransformerMixin, BaseEstimator):
 
     .. note::
        This structure only can be used for CAFFE pretrained model.
-
-
     """
 
     def __init__(self, **kwargs):
@@ -82,10 +80,10 @@ class opencv_model(TransformerMixin, BaseEstimator):
     """
 
         if self.model is None:
-
             self._load_model()
 
         img = np.array(X)
+        img = img / 255
 
         self.model.setInput(img)
 
@@ -99,5 +97,4 @@ class opencv_model(TransformerMixin, BaseEstimator):
         return d
 
     def _more_tags(self):
-
         return {"stateless": True, "requires_fit": False}

bob/bio/face/extractor/pytorch_model.py → bob/bio/face/extractor/PyTorchModel.py

@@ -17,7 +17,7 @@ pytorch_model_directory = rc["bob.extractor_model.pytorch"]
 pytorch_weight_directory = rc["bob.extractor_weights.pytorch"]
 
 
-class pytorch_loaded_model(TransformerMixin, BaseEstimator):
+class PyTorchLoadedModel(TransformerMixin, BaseEstimator):
     """Extracts features using deep face recognition models under PyTorch Interface, especially for the models and weights that need to load by hand.
     
   Users can download the pretrained face recognition models with PyTorch Interface. The path to downloaded models should be specified before running the extractor (usually before running the pipeline file that includes the extractor). That is, set config of the model frame to :py:class:`bob.extractor_model.pytorch`, and set config of the parameters to :py:class:`bob.extractor_weights.pytorch`. 
@@ -28,8 +28,6 @@ class pytorch_loaded_model(TransformerMixin, BaseEstimator):
     $ bob config set bob.extractor_weights.pytorch /PATH/TO/WEIGHTS/
   
   The extracted features can be combined with different the algorithms. 
-
-
     """
 
     def __init__(self, **kwargs):
@@ -76,11 +74,11 @@ class pytorch_loaded_model(TransformerMixin, BaseEstimator):
     feature : 2D or 3D :py:class:`numpy.ndarray` (floats)
       The list of features extracted from the image.
     """
-
         if self.model is None:
             self._load_model()
 
         X = torch.Tensor(X)
+        X = X / 255
 
         return self.model(X).detach().numpy()
 
@@ -92,11 +90,10 @@ class pytorch_loaded_model(TransformerMixin, BaseEstimator):
         return d
 
     def _more_tags(self):
-
         return {"stateless": True, "requires_fit": False}
 
 
-class pytorch_library_model(TransformerMixin, BaseEstimator):
+class PyTorchLibraryModel(TransformerMixin, BaseEstimator):
     """Extracts features using deep face recognition with registered model frames in the PyTorch Library. 
     
   Users can import the pretrained face recognition models from PyTorch library. The model should be called in the pipeline. Example: `facenet_pytorch <https://github.com/timesler/facenet-pytorch>`_
@@ -105,7 +102,6 @@ class pytorch_library_model(TransformerMixin, BaseEstimator):
 
   **Parameters:**
   model: pytorch model calling from library.
-  use_gpu: True or False.
     """
 
     def __init__(self, model=None, **kwargs):
@@ -125,6 +121,11 @@ class pytorch_library_model(TransformerMixin, BaseEstimator):
         self.checkpoint_path = checkpoint_path
         self.device = None
 
+    def _load_model(self):
+
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model.to(self.device)
+
     def transform(self, X):
         """__call__(image) -> feature
 
@@ -141,7 +142,11 @@ class pytorch_library_model(TransformerMixin, BaseEstimator):
       The list of features extracted from the image.
     """
 
+        if self.model is None:
+            self._load_model()
+
         X = torch.Tensor(X)
+        X = X / 255
 
         return self.model(X).detach().numpy()
 

bob/bio/face/extractor/tf_model.py → bob/bio/face/extractor/TensorFlowModel.py

@@ -18,7 +18,7 @@ from tensorflow import keras
 tf_model_directory = rc["bob.extractor_model.tf"]
 
 
-class tf_model(TransformerMixin, BaseEstimator):
+class TensorFlowModel(TransformerMixin, BaseEstimator):
     """Extracts features using deep face recognition models under TensorFlow Interface.
 
   Users can download the pretrained face recognition models with TensorFlow Interface. The path to downloaded models should be specified before running the extractor (usually before running the pipeline file that includes the extractor). That is, set config of the model to :py:class:`bob.extractor_model.tf`. 
@@ -28,8 +28,6 @@ class tf_model(TransformerMixin, BaseEstimator):
     $ bob config set bob.extractor_model.tf /PATH/TO/MODEL/
   
   The extracted features can be combined with different the algorithms. 
-
-
     """
 
     def __init__(self, **kwargs):
@@ -76,6 +74,8 @@ class tf_model(TransformerMixin, BaseEstimator):
         X = check_array(X, allow_nd=True)
         X = tf.convert_to_tensor(X)
         X = to_channels_last(X)
+
+        X = X / 255
         predict = self.model.predict(X)
 
         return predict

bob/bio/face/extractor/__init__.py

 from .DCTBlocks import DCTBlocks
 from .GridGraph import GridGraph
 from .LGBPHS import LGBPHS
-from .mxnet_resnet import mxnet_model
-from .pytorch_model import pytorch_loaded_model
-from .pytorch_model import pytorch_library_model
-from .tf_model import tf_model
-from .opencv_caffe import opencv_model
+from .MxNetModel import MxNetModel
+from .PyTorchModel import PyTorchLoadedModel
+from .PyTorchModel import PyTorchLibraryModel
+from .TensorFlowModel import TensorFlowModel
+from .OpenCVModel import OpenCVModel
 
 # gets sphinx autodoc done right - don't remove it
 def __appropriate__(*args):
@@ -27,10 +27,10 @@ __appropriate__(
     DCTBlocks,
     GridGraph,
     LGBPHS,
-    mxnet_model,
-    pytorch_loaded_model,
-    pytorch_library_model,
-    tf_model,
-    opencv_model,
+    MxNetModel,
+    PyTorchLoadedModel,
+    PyTorchLibraryModel,
+    TensorFlowModel,
+    OpenCVModel,
 )
 __all__ = [_ for _ in dir() if not _.startswith("_")]

doc/baselines.rst

@@ -61,12 +61,14 @@ Deep learning baselines
 Deep Learning with different interfaces baselines
 =================================================
 
-* ``mxnet_pipe``: Arcface Resnet Model using MxNet Interfaces from `Insightface <https://github.com/deepinsight/insightface>`_
+* ``mxnet-pipe``: Arcface Resnet Model using MxNet Interfaces from `Insightface <https://github.com/deepinsight/insightface>`_
 
-* ``pytorch_pipe_v1``: Pytorch network that extracs 1000-dimensional featrues, trained by Manual Gunther, as described in [LGB18]_
+* ``mxnet-tinyface``: Applying `tinyface annoator <https://github.com/chinakook/hr101_mxnet>`_ for the Arcface Resnet Model using MxNet Interfaces from `Insightface <https://github.com/deepinsight/insightface>`_
 
-* ``pytorch_pipe_v2``: Inception Resnet face recognition model from `facenet_pytorch <https://github.com/timesler/facenet-pytorch>`_
+* ``pytorch-pipe-v1``: Pytorch network that extracts 1000-dimensional features, trained by Manual Gunther, as described in [LGB18]_
 
-* ``tf_pipe``: Inception Resnet v2 model trained using the MSCeleb dataset in the context of the work published by [TFP18]_
+* ``pytorch-pipe-v2``: Inception Resnet face recognition model from `facenet_pytorch <https://github.com/timesler/facenet-pytorch>`_
 
-* ``opencv_pipe``: VGG Face descriptor pretrained models, i.e. `Caffe model <https://www.robots.ox.ac.uk/~vgg/software/vgg_face/>`_
+* ``tf-pipe``: Inception Resnet v2 model trained using the MSCeleb dataset in the context of the work published by [TFP18]_
+
+* ``opencv-pipe``: VGG Face descriptor pretrained models, i.e. `Caffe model <https://www.robots.ox.ac.uk/~vgg/software/vgg_face/>`_

doc/deeplearningextractor.rst

+.. vim: set fileencoding=utf-8 :
+.. author: Yu Linghu & Xinyi Zhang <yu.linghu@uzh.ch, xinyi.zhang@uzh.ch>
+
+.. _bob.bio.face.deeplearningextractor:
+
+==========================================================================
+Extractors using Pretrained Deep Neural Networks with Different Interfaces
+==========================================================================
+
+
+In this page we introduce how to import and use the pretrained feature-extraction models in a face recognition experiment.
+
+
+
+Extractor 
+---------
+
+As explained in :ref:`bob.bio.base <bob.bio.base>`, the extractor is a transformer that can reduce the dimensionality of preprocessed images so that the resulting feature vectors can ease the classification procedures (calculating the differences between individual’s vectors).  
+ 
+
+ 
+Pretrained Models as Extractors
+-------------------------------
+
+The typical extractors like ``Discrete Cosine Transform (DCT)``, ``Gabor jets in a grid structure`` [GHW12]_, and ``Local Gabor Binary Pattern Histogram Sequences (LGBPHS)`` [ZSG05]_ are used before the deep learning era. To adapt to the deep learning era, the extractors introduced in this page are used to import the pretrained feature extraction models and weights. In other words, we use the preprocessed images as the inputs, import the pretrained model and its corresponding  weights, and plug inputs into the model to return the feature vectors. In this way, users can use their own or downloaded pretrained models to capture features. 
+ 
+.. note::
+  The following Interfaces are supported:
+  
+  MxNet
+  
+  PyTorch: Either import models by hand or call from Library
+  
+  TensorFlow
+  
+  OpenCV
+  
+  
+  Please check the implementation details below.
+
+
+
+Step By Step Instructions
+-------------------------
+
+
+1.   Download your desired model framework and pretrained weights.
+
+2.   Specify the path to models before setting up the experiment configurations. 
+    
+    MxNet
+       
+    .. code-block:: sh
+   
+        $ bob config set bob.extractor_model.mxnet /PATH/TO/MODEL/
+        $ bob config set bob.extractor_weights.mxnet /PATH/TO/WEIGHTS/
+
+    OpenCV:
+       
+    .. code-block:: sh
+   
+        $ bob config set bob.extractor_model.opencv /PATH/TO/MODEL/
+        $ bob config set bob.extractor_weights.opencv /PATH/TO/WEIGHTS/
+
+    PyTorch:
+       
+    .. code-block:: sh
+   
+        $ bob config set bob.extractor_model.pytorch /PATH/TO/MODEL/
+        $ bob config set bob.extractor_weights.pytorch /PATH/TO/WEIGHTS/
+       
+    PyTorch (Call from Library): No need to set the path, please define the model within the configuration file.
+       
+    TensorFlow:
+       
+    .. code-block:: sh
+   
+        $ bob config set bob.extractor_model.tf /PATH/TO/MODEL/
+
+3.   Call the extractor in the configuration file. Example for MxNet:
+       
+    .. code-block:: sh
+   
+        from bob.bio.face.extractor import MxNetModel
+        extractor_transformer = MxNetModel()
+
+4.   Run the pipeline as usual.
+
+
+ 
+Baselines
+---------
+
+The available baselines with the resulting ROC plots  for each interface are listed below.
+
+
+MxNet
+=====
+*  ``mxnet-pipe``: Arcface Resnet Model using MxNet Interfaces from `Insightface <https://github.com/deepinsight/insightface>`_
+
+1.   Set Path of Model
+       
+    .. code-block:: sh
+   
+        $ bob config set bob.extractor_model.mxnet /PATH/TO/MXNETMODEL/
+        $ bob config set bob.extractor_weights.mxnet /PATH/TO/MXNETWEIGHTS/
+
+2.   Call the extractor in the configuration
+       
+    .. code-block:: sh
+   
+        from bob.bio.face.extractor import MxNetModel
+        extractor_transformer = MxNetModel()
+
+In this baseline, we use :py:class:`bob.bio.face.preprocessor.FaceCrop` with  ``cropped_positions={'leye':(49,72), 'reye':(49,38)}`` 
+and ``cropped_image_size=(112,112)`` 
+as preprocessor,  `LResNet50E-IR,ArcFace@ms1m-refine-v1 <https://github.com/deepinsight/insightface/wiki/Model-Zoo#32-lresnet50e-irarcfacems1m-refine-v1>`_ as extractor and ``distance-cosine`` as the algorithm. By testing on LFW database, we get the following ROC plot:
+
+.. figure:: img/mxnet_lfw_pipe.png
+  :figwidth: 75%
+  :align: center
+  :alt: Face recognition results of LFW database.
+
+  ROC plot for LFW database using the pretrained extractor with MxNet Interface.
+
+
+OpenCV
+======
+* ``opencv-pipe``: VGG Face descriptor pretrained models, i.e. `Caffe model <https://www.robots.ox.ac.uk/~vgg/software/vgg_face/>`_
+
+1.   Set Path of Model
+