Commit e7aa5899 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Add MTCNN (tensorflow) model

parent c4e980f9
Pipeline #29833 passed with stage
in 22 minutes and 24 seconds
......@@ -12,3 +12,6 @@ develop-eggs
sphinx
dist
bob/ip/tensorflow_extractor/data/FaceNet/
record.txt
build/
bob/ip/tensorflow_extractor/data/DR_GAN_model/
......@@ -63,12 +63,12 @@ class Extractor(object):
Parameters
----------
image : numpy.array
image : numpy.ndarray
Input Data
Returns
-------
numpy.array
numpy.ndarray
The features.
"""
......
# code and model from https://github.com/blaueck/tf-mtcnn
import pkg_resources
import tensorflow as tf
import multiprocessing
import bob.io.image
MODEL_PATH = pkg_resources.resource_filename(__name__, "data/mtcnn/mtcnn.pb")
class MTCNN:
"""MTCNN v1 wrapper. See
https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for more
details on MTCNN and see :ref:`bob.ip.tensorflow_extractor.face_detect` for an
example code.
Attributes
----------
factor : float
Factor is a trade-off between performance and speed.
min_size : int
Minimum face size to be detected.
thresholds : list
thresholds are a trade-off between false positives and missed detections.
"""
def __init__(
self,
min_size=40,
factor=0.709,
thresholds=(0.6, 0.7, 0.7),
model_path=MODEL_PATH,
):
self.min_size = min_size
self.factor = factor
self.thresholds = thresholds
graph = tf.Graph()
with graph.as_default():
with open(model_path, "rb") as f:
graph_def = tf.GraphDef.FromString(f.read())
tf.import_graph_def(graph_def, name="")
self.graph = graph
config = tf.ConfigProto(
intra_op_parallelism_threads=multiprocessing.cpu_count(),
inter_op_parallelism_threads=multiprocessing.cpu_count(),
)
self.sess = tf.Session(graph=graph, config=config)
def detect(self, img):
"""Detects all faces in the image.
Parameters
----------
img : numpy.ndarray
An RGB image in Bob format.
Returns
-------
tuple
A tuple of boxes, probabilities, and landmarks.
"""
# assuming img is Bob format and RGB
assert img.shape[0] == 3, img.shape
# network expects BGR opencv format
img = bob.io.image.to_matplotlib(img)
img = img[..., ::-1]
feeds = {
self.graph.get_operation_by_name("input").outputs[0]: img,
self.graph.get_operation_by_name("min_size").outputs[0]: self.min_size,
self.graph.get_operation_by_name("thresholds").outputs[0]: self.thresholds,
self.graph.get_operation_by_name("factor").outputs[0]: self.factor,
}
fetches = [
self.graph.get_operation_by_name("prob").outputs[0],
self.graph.get_operation_by_name("landmarks").outputs[0],
self.graph.get_operation_by_name("box").outputs[0],
]
prob, landmarks, box = self.sess.run(fetches, feeds)
return box, prob, landmarks
def annotations(self, img):
"""Detects all faces in the image
Parameters
----------
img : numpy.ndarray
An RGB image in Bob format.
Returns
-------
list
A list of annotations. Annotations are dictionaries that contain the
following keys: ``topleft``, ``bottomright``, ``reye``, ``leye``, ``nose``,
``right_of_mouth``, ``left_of_mouth``, and ``quality``.
"""
boxes, scores, landmarks = self.detect(img)
annots = []
for box, prob, lm in zip(boxes, scores, landmarks):
topleft = box[0], box[1]
bottomright = box[2], box[3]
right_eye = lm[0], lm[5]
left_eye = lm[1], lm[6]
nose = lm[2], lm[7]
right_of_mouth = lm[3], lm[8]
left_of_mouth = lm[4], lm[9]
annots.append(
{
"topleft": topleft,
"bottomright": bottomright,
"reye": right_eye,
"leye": left_eye,
"nose": nose,
"right_of_mouth": right_of_mouth,
"left_of_mouth": left_of_mouth,
"quality": prob,
}
)
return annots
def __call__(self, img):
"""Wrapper for the annotations method.
"""
return self.annotations(img)
......@@ -37,6 +37,7 @@ from .Extractor import Extractor
from .FaceNet import FaceNet
from .DrGanMSU import DrGanMSUExtractor
from .Vgg16 import VGGFace, vgg_16
from .MTCNN import MTCNN
# gets sphinx autodoc done right - don't remove it
......@@ -59,7 +60,8 @@ __appropriate__(
Extractor,
FaceNet,
DrGanMSUExtractor,
VGGFace
VGGFace,
MTCNN,
)
# gets sphinx autodoc done right - don't remove it
......
[
{
"topleft": [
40.24329,
113.20566
],
"bottomright": [
106.32423,
162.45758
],
"reye": [
67.90105,
124.33353
],
"leye": [
64.25375,
145.77344
],
"nose": [
81.764984,
136.25734
],
"right_of_mouth": [
88.05158,
128.52052
],
"left_of_mouth": [
84.64868,
150.7494
],
"quality": 0.99999917
},
{
"topleft": [
34.18492,
427.5858
],
"bottomright": [
94.81634,
471.9476
],
"reye": [
56.84209,
446.76434
],
"leye": [
64.74565,
462.9847
],
"nose": [
71.690926,
454.36282
],
"right_of_mouth": [
75.266556,
438.5781
],
"left_of_mouth": [
82.32741,
454.99423
],
"quality": 0.99981314
},
{
"topleft": [
69.87796,
31.797615
],
"bottomright": [
123.43042,
78.04486
],
"reye": [
96.72684,
43.90791
],
"leye": [
89.42775,
61.957954
],
"nose": [
104.02164,
56.380474
],
"right_of_mouth": [
111.99057,
49.277725
],
"left_of_mouth": [
104.45787,
69.476105
],
"quality": 0.9994398
},
{
"topleft": [
105.99489,
238.27567
],
"bottomright": [
159.2981,
280.56006
],
"reye": [
125.97672,
249.38193
],
"leye": [
127.117195,
268.7624
],
"nose": [
138.73158,
257.6499
],
"right_of_mouth": [
142.67319,
246.85234
],
"left_of_mouth": [
143.5627,
267.6805
],
"quality": 0.99918956
},
{
"topleft": [
48.377903,
321.892
],
"bottomright": [
110.95402,
367.87064
],
"reye": [
73.381096,
334.53403
],
"leye": [
76.274086,
355.40384
],
"nose": [
86.6857,
344.3223
],
"right_of_mouth": [
94.80564,
331.12646
],
"left_of_mouth": [
96.63391,
351.96518
],
"quality": 0.9987685
},
{
"topleft": [
115.29803,
159.48656
],
"bottomright": [
172.41876,
205.56857
],
"reye": [
141.30688,
171.35336
],
"leye": [
137.49718,
191.17722
],
"nose": [
151.32994,
182.92662
],
"right_of_mouth": [
159.72272,
175.2344
],
"left_of_mouth": [
156.25536,
193.2938
],
"quality": 0.99671644
}
]
import bob.io.base
import bob.io.image
from bob.io.base.test_utils import datafile
import bob.ip.tensorflow_extractor
import tensorflow as tf
import pkg_resources
import numpy
numpy.random.seed(10)
import json
import os
numpy.random.seed(10)
slim = tf.contrib.slim
from . import scratch_network
......@@ -15,8 +18,9 @@ from . import scratch_network
def test_output():
# Loading MNIST model
filename = os.path.join(pkg_resources.resource_filename(
__name__, 'data'), 'model.ckp')
filename = os.path.join(
pkg_resources.resource_filename(__name__, "data"), "model.ckp"
)
inputs = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
# Testing the last output
......@@ -41,23 +45,52 @@ def test_output():
def test_facenet():
from bob.ip.tensorflow_extractor import FaceNet
extractor = FaceNet()
data = numpy.random.rand(3, 160, 160).astype("uint8")
output = extractor(data)
assert output.size == 128, output.shape
def test_drgan():
from bob.ip.tensorflow_extractor import DrGanMSUExtractor
extractor = DrGanMSUExtractor()
data = numpy.random.rand(3, 96, 96).astype("uint8")
output = extractor(data)
assert output.size == 320, output.shape
def test_vgg16():
pass
#from bob.ip.tensorflow_extractor import VGGFace
#extractor = VGGFace()
#data = numpy.random.rand(3, 224, 224).astype("uint8")
#output = extractor(data)
#assert output.size == 4096, output.shape
# from bob.ip.tensorflow_extractor import VGGFace
# extractor = VGGFace()
# data = numpy.random.rand(3, 224, 224).astype("uint8")
# output = extractor(data)
# assert output.size == 4096, output.shape
def test_mtcnn():
test_image = datafile("mtcnn/test_image.png", __name__)
ref_numbers = datafile("mtcnn/mtcnn.hdf5", __name__)
ref_annots = datafile("mtcnn/mtcnn.json", __name__)
from bob.ip.tensorflow_extractor import MTCNN
mtcnn = MTCNN()
img = bob.io.base.load(test_image)
bbox, prob, landmarks = mtcnn.detect(img)
with bob.io.base.HDF5File(ref_numbers, "r") as f:
ref_bbox = f["bbox"]
ref_scores = f["scores"]
ref_landmarks = f["landmarks"]
assert numpy.allclose(bbox, ref_bbox), (bbox, ref_bbox)
assert numpy.allclose(prob, ref_scores), (prob, ref_scores)
assert numpy.allclose(landmarks, ref_landmarks), (landmarks, ref_landmarks)
annots = mtcnn.annotations(img)
ref_annots = json.load(open(ref_annots))
for a, aref in zip(annots, ref_annots):
for k, v in a.items():
vref = aref[k]
assert numpy.allclose(v, vref)
......@@ -25,12 +25,14 @@ requirements:
- bob.db.mnist
- bob.ip.color
- six {{ six }}
- tensorflow {{ tensorflow }}
- numpy {{ numpy }}
run:
- python
- setuptools
- scipy
- six
- tensorflow >=1.2.1
- {{ pin_compatible('tensorflow') }}
- {{ pin_compatible('numpy') }}
test:
imports:
......@@ -47,7 +49,6 @@ test:
- coverage
- sphinx
- sphinx_rtd_theme
- bob.io.image
- bob.db.atnt
- matplotlib
- gridtk
......
......@@ -36,7 +36,7 @@ else:
extensions.append('sphinx.ext.pngmath')
# Be picky about warnings
nitpicky = False
nitpicky = True
# Ignores stuff we can't easily resolve on other project's sphinx manuals
nitpick_ignore = []
......
.. _bob.ip.tensorflow_extractor.face_detect:
============================
Face detection using MTCNN
============================
This package comes with a wrapper around the MTCNN (v1) face detector. See
https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for more
information on MTCNN. The model is directly converted from the caffe model using code in
https://github.com/blaueck/tf-mtcnn
See below for an example on how to use
:any:`bob.ip.tensorflow_extractor.MTCNN`:
.. plot:: plot/detect_faces_mtcnn.py
:include-source: True
......@@ -16,4 +16,5 @@ Tensorflow http://tensorflow.org/
:maxdepth: 2
guide
face_detect
py_api
from bob.io.image import imshow
from bob.io.base import load
from bob.io.base.test_utils import datafile
from bob.ip.tensorflow_extractor import MTCNN
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, Circle
# load colored test image
color_image = load(datafile("mtcnn/test_image.png", "bob.ip.tensorflow_extractor"))
# detect all face
detector = MTCNN()
detections = detector(color_image)
imshow(color_image)
plt.axis("off")
for annotations in detections:
topleft = annotations["topleft"]
bottomright = annotations["bottomright"]
size = bottomright[0] - topleft[0], bottomright[1] - topleft[1]
# draw bounding boxes
plt.gca().add_patch(
Rectangle(
topleft[::-1],
size[1],
size[0],
edgecolor="b",
facecolor="none",
linewidth=2,
)
)
# draw face landmarks
for key, color in (
("reye", "r"),
("leye", "g"),
("nose", "b"),
("right_of_mouth", "k"),
("left_of_mouth", "w"),
):
plt.gca().add_patch(
Circle(
annotations[key][::-1],
radius=2,
edgecolor=color,
facecolor="none",
linewidth=2,
)
)
# show quality of detections
plt.text(
topleft[1], topleft[0], round(annotations["quality"], 3), color="b", fontsize=14
)
......@@ -9,6 +9,7 @@ Classes
.. autosummary::
bob.ip.tensorflow_extractor.Extractor
bob.ip.tensorflow_extractor.FaceNet
bob.ip.tensorflow_extractor.MTCNN
Detailed API
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment