Merge branch 'tf2-mtcnn-graph' into 'master'

Port mtcnn to Tensorflow 2

See merge request !13

MANIFEST.in

 include README.rst bootstrap-buildout.py buildout.cfg develop.cfg COPYING requirements.txt version.txt
 recursive-include doc doc/plot doc/img *.py *.rst *.png *.ico
 recursive-include bob/ip/facedetect *.cpp *.h
-recursive-include bob/ip/facedetect/data *.jpg *.pos *.hdf5
+recursive-include bob/ip/facedetect/data *.jpg *.pos *.hdf5 *.png *.pb
 include bob/ip/facedetect/MCT_cascade.hdf5
 recursive-include bob/learn/boosting *.h *.cpp
 recursive-include bob/learn/boosting/data *.hdf5 *.tar.bz2

bob/ip/facedetect/__init__.py

@@ -12,7 +12,6 @@ from .train import *
 
 from .detect import default_cascade, average_detections, best_detection, detect_single_face, detect_all_faces
 
-
 def get_config():
   """Returns a string containing the configuration information.
   """

bob/ip/facedetect/mtcnn.py

+# Example taken from:
+# https://github.com/blaueck/tf-mtcnn/blob/master/mtcnn_tfv2.py
+
+import logging
+
+import pkg_resources
+from bob.io.image import to_matplotlib
+from bob.ip.color import gray_to_rgb
+
+logger = logging.getLogger(__name__)
+
+
+class MTCNN:
+
+    """MTCNN v1 wrapper for Tensorflow 2. See
+    https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for
+    more details on MTCNN.
+
+    Attributes
+    ----------
+    factor : float
+        Factor is a trade-off between performance and speed.
+    min_size : int
+        Minimum face size to be detected.
+    thresholds : list
+        Thresholds are a trade-off between false positives and missed detections.
+    """
+
+    def __init__(self, min_size=40, factor=0.709, thresholds=(0.6, 0.7, 0.7), **kwargs):
+        super().__init__(**kwargs)
+        self.min_size = min_size
+        self.factor = factor
+        self.thresholds = thresholds
+        self._graph_path = pkg_resources.resource_filename(__name__, "data/mtcnn.pb")
+
+        # Avoids loading graph at initilization
+        self._fun = None
+
+    @property
+    def mtcnn_fun(self):
+        import tensorflow as tf
+
+        if self._fun is None:
+            # wrap graph function as a callable function
+            self._fun = tf.compat.v1.wrap_function(
+                self._graph_fn,
+                [
+                    tf.TensorSpec(shape=[None, None, 3], dtype=tf.float32),
+                ],
+            )
+        return self._fun
+
+    def _graph_fn(self, img):
+        import tensorflow as tf
+
+        with open(self._graph_path, "rb") as f:
+            graph_def = tf.compat.v1.GraphDef.FromString(f.read())
+
+        prob, landmarks, box = tf.compat.v1.import_graph_def(
+            graph_def,
+            input_map={
+                "input:0": img,
+                "min_size:0": tf.convert_to_tensor(self.min_size, dtype=float),
+                "thresholds:0": tf.convert_to_tensor(self.thresholds, dtype=float),
+                "factor:0": tf.convert_to_tensor(self.factor, dtype=float),
+            },
+            return_elements=["prob:0", "landmarks:0", "box:0"],
+            name="",
+        )
+        return box, prob, landmarks
+
+    def detect(self, image):
+        """Detects all faces in the image.
+
+        Parameters
+        ----------
+        image : numpy.ndarray
+            An RGB image in Bob format.
+
+        Returns
+        -------
+        tuple
+            A tuple of boxes, probabilities, and landmarks.
+        """
+        if len(image.shape) == 2:
+            image = gray_to_rgb(image)
+
+        # Assuming image is Bob format and RGB
+        assert image.shape[0] == 3, image.shape
+        # MTCNN expects BGR opencv format
+        image = to_matplotlib(image)
+        image = image[..., ::-1]
+
+        boxes, probs, landmarks = self.mtcnn_fun(image)
+        return boxes, probs, landmarks
+
+    def annotations(self, image):
+        """Detects all faces in the image and returns annotations in bob format.
+
+        Parameters
+        ----------
+        image : numpy.ndarray
+            An RGB image in Bob format.
+
+        Returns
+        -------
+        list
+            A list of annotations. Annotations are dictionaries that contain the
+            following keys: ``topleft``, ``bottomright``, ``reye``, ``leye``, ``nose``,
+            ``mouthright``, ``mouthleft``, and ``quality``.
+        """
+        boxes, probs, landmarks = self.detect(image)
+
+        # Iterate over all the detected faces
+        annots = []
+        for box, prob, lm in zip(boxes, probs, landmarks):
+            topleft = float(box[0]), float(box[1])
+            bottomright = float(box[2]), float(box[3])
+            right_eye = float(lm[0]), float(lm[5])
+            left_eye = float(lm[1]), float(lm[6])
+            nose = float(lm[2]), float(lm[7])
+            mouthright = float(lm[3]), float(lm[8])
+            mouthleft = float(lm[4]), float(lm[9])
+            annots.append(
+                {
+                    "topleft": topleft,
+                    "bottomright": bottomright,
+                    "reye": right_eye,
+                    "leye": left_eye,
+                    "nose": nose,
+                    "mouthright": mouthright,
+                    "mouthleft": mouthleft,
+                    "quality": float(prob),
+                }
+            )
+        return annots
+
+    def __call__(self, img):
+        """Wrapper for the annotations method."""
+        return self.annotations(img)

bob/ip/facedetect/tests/test_mtcnn.py

+
+from bob.ip.facedetect.tests.utils import is_library_available
+
+import bob.io.image
+import bob.io.base
+import bob.io.base.test_utils
+
+import numpy
+
+
+# An image with one face
+face_image = bob.io.base.load(
+    bob.io.base.test_utils.datafile(
+        'testimage.jpg', 'bob.ip.facedetect'
+    )
+)
+
+# An image with 6 faces
+face_image_multiple = bob.io.base.load(
+    bob.io.base.test_utils.datafile(
+        'test_image_multi_face.png', 'bob.ip.facedetect'
+    )
+)
+
+
+def _assert_mtcnn_annotations(annot):
+    """
+    Verifies that MTCNN returns the correct coordinates for ``testimage``.
+    """
+    assert len(annot) == 1, f"len: {len(annot)}; {annot}"
+    face = annot[0]
+    assert [int(x) for x in face['topleft']] == [68, 76], face
+    assert [int(x) for x in face['bottomright']] == [344, 274], face
+    assert [int(x) for x in face['reye']] == [180, 129], face
+    assert [int(x) for x in face['leye']] == [175, 220], face
+    assert numpy.allclose(face['quality'], 0.9998974), face
+
+@is_library_available("tensorflow")
+def test_mtcnn():
+    """MTCNN should annotate one face correctly."""
+    from bob.ip.facedetect.mtcnn import MTCNN
+    mtcnn_annotator = MTCNN()
+    annot = mtcnn_annotator.annotations(face_image)
+    _assert_mtcnn_annotations(annot)
+
+@is_library_available("tensorflow")
+def test_mtcnn_multiface():
+    """MTCNN should find multiple faces in an image."""
+    from bob.ip.facedetect.mtcnn import MTCNN
+    mtcnn_annotator = MTCNN()
+    annot = mtcnn_annotator.annotations(face_image_multiple)
+    assert len(annot) == 6

bob/ip/facedetect/tests/utils.py

+from nose.plugins.skip import SkipTest
+import functools
+import importlib
+
+def is_library_available(library):
+    """Decorator to check if a library is present, before running that test"""
+
+    def _is_library_available(function):
+        @functools.wraps(function)
+        def wrapper(*args, **kwargs):
+            try:
+                importlib.import_module(library)
+
+                return function(*args, **kwargs)
+            except ImportError as e:
+                raise SkipTest(
+                    f"Skipping test since `{library}` is not available: %s" % e
+                )
+
+        return wrapper
+
+    return _is_library_available

conda/meta.yaml

@@ -40,15 +40,18 @@ requirements:
     - boost {{ boost }}
     - numpy {{ numpy }}
     - scipy {{ scipy }}
+    - matplotlib {{ matplotlib }}
+    - tensorflow {{ tensorflow }}  # [linux]
   run:
     - python
     - setuptools
     - boost
-    - scipy
-    - scikit-image
+    - {{ pin_compatible('scipy') }}
+    - {{ pin_compatible('scikit-image') }}
     - {{ pin_compatible('numpy') }}
   run_constrained:
-    - matplotlib
+    - {{ pin_compatible('matplotlib') }}
+    - {{ pin_compatible('tensorflow') }}  # [linux]
 
 test:
   imports:
@@ -75,6 +78,7 @@ test:
     - sphinx
     - sphinx_rtd_theme
     - matplotlib
+    - tensorflow  # [linux]
 
 about:
   home: https://www.idiap.ch/software/bob/

doc/index.rst

@@ -34,6 +34,7 @@ Documentation
    :maxdepth: 2
 
    guide
+   mtcnn
    py_api
 
 

doc/mtcnn.rst

+
+.. _bob.ip.facedetect.mtcnn:
+
+============================
+ Face detection using MTCNN
+============================
+
+This package comes with a wrapper around the MTCNN (v1) face detector. See
+https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for more
+information on MTCNN. The model is directly converted from the caffe model using code in
+https://github.com/blaueck/tf-mtcnn
+
+See below for an example on how to use
+:any:`bob.ip.facedetect.mtcnn.MTCNN`:
+
+.. plot:: plot/detect_faces_mtcnn.py
+   :include-source: True
+

doc/plot/detect_faces_mtcnn.py

+import matplotlib.pyplot as plt
+from bob.io.base import load
+from bob.io.base.test_utils import datafile
+from bob.io.image import imshow
+from bob.ip.facedetect.mtcnn import MTCNN
+from matplotlib.patches import Circle
+from matplotlib.patches import Rectangle
+
+# load colored test image
+color_image = load(datafile("test_image_multi_face.png", "bob.ip.facedetect"))
+is_tf_available = True
+try:
+    import tensorflow
+except Exception:
+    is_tf_available = False
+
+if not is_tf_available:
+    imshow(color_image)
+else:
+
+    # detect all face
+    detector = MTCNN()
+    detections = detector(color_image)
+
+    imshow(color_image)
+    plt.axis("off")
+
+    for annotations in detections:
+        topleft = annotations["topleft"]
+        bottomright = annotations["bottomright"]
+        size = bottomright[0] - topleft[0], bottomright[1] - topleft[1]
+        # draw bounding boxes
+        plt.gca().add_patch(
+            Rectangle(
+                topleft[::-1],
+                size[1],
+                size[0],
+                edgecolor="b",
+                facecolor="none",
+                linewidth=2,
+            )
+        )
+        # draw face landmarks
+        for key, color in (
+            ("reye", "r"),
+            ("leye", "g"),
+            ("nose", "b"),
+            ("mouthright", "k"),
+            ("mouthleft", "w"),
+        ):
+            plt.gca().add_patch(
+                Circle(
+                    annotations[key][::-1],
+                    radius=2,
+                    edgecolor=color,
+                    facecolor="none",
+                    linewidth=2,
+                )
+            )
+        # show quality of detections
+        plt.text(
+            topleft[1],
+            topleft[0],
+            round(annotations["quality"], 3),
+            color="b",
+            fontsize=14,
+        )

doc/py_api.rst

@@ -40,3 +40,4 @@ Detailed Information
 --------------------
 
 .. automodule:: bob.ip.facedetect
+.. automodule:: bob.ip.facedetect.mtcnn