Commit 5e235e4a authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira

Merge branch 'dev_branch' into 'master'

Fixed possible negative coordinates in the face bbx, updated the documentation

See merge request !1
parents 6c50c4e8 be66cf41
Pipeline #13680 passed with stages
in 9 minutes and 53 seconds
......@@ -19,16 +19,16 @@ from .utils import bob_to_dlib_image_convertion, rectangle2bounding_box2
class FaceDetector(object):
"""
Detects face and 5 landmarks using the MTCNN (https://github.com/kpzhang93/MTCNN_face_detection_alignment) from the paper.
Zhang, Kaipeng, et al. "Joint face detection and alignment using multitask cascaded convolutional networks." IEEE Signal Processing Letters 23.10 (2016): 1499-1503.
"""
def __init__(self):
"""
Load the caffe models
"""
caffe_base_path = FaceDetector.get_mtcnn_model_path()
# Default value from the example
......@@ -36,7 +36,7 @@ class FaceDetector(object):
self.threshold = [0.6, 0.7, 0.7]
self.factor = 0.709
self.fastresize = False
# Loading the models
caffe.set_mode_cpu()
self.p_net = caffe.Net(os.path.join(caffe_base_path, "det1.prototxt"), os.path.join(caffe_base_path, "det1.caffemodel"), caffe.TEST)
......@@ -56,8 +56,8 @@ class FaceDetector(object):
for offset, p in zip(range(len(possible_landmarks)), possible_landmarks):
l[p] = ( int(points[i][offset+5]), int(points[i][offset]) )
landmarks.append(l)
return landmarks
return landmarks
def detect_all_faces(self, image, return_bob_bb = True):
......@@ -70,29 +70,29 @@ class FaceDetector(object):
return_bob_bb: if true will return the faces wrapped using py:class:`bob.ip.facedetect.BoundingBox`
**Returns**
Returns two lists; the first on contains the bounding boxes with the detected faces and the second one
contains list with the faces landmarks. The CNN returns 5 facial landmarks
(leye, reye, nose, mouthleft, mouthright). If there's no face, `None` will be returned
"""
assert image is not None
if len(image.shape) !=3:
raise ValueError("Only color images is supported")
bb, landmarks = detect_face(bob_to_dlib_image_convertion(image), self.minsize, self.p_net, self.r_net, self.o_net, self.threshold, self.fastresize, self.factor)
# If there's no face, return none
if len(bb) == 0:
return None, None
if return_bob_bb:
bb = rectangle2bounding_box2(bb)
return bb, self._convert_list_to_landmarks(landmarks)
def detect_single_face(self, image):
"""
Returns the biggest face in a COLORED image, if any.
......@@ -104,14 +104,14 @@ class FaceDetector(object):
**Returns**
The face bounding box and its respective 5 landmarks (leye, reye, nose, mouthleft, mouthright).
If there's no face, `None` will be returned
"""
faces, landmarks = self.detect_all_faces(image)
# Return None if
if faces is None:
return None, None
index = numpy.argmax([(f.bottomright[0] - f.topleft[0]) * (f.bottomright[1] - f.topleft[1]) for f in faces])
return faces[index], landmarks[index]
......@@ -120,56 +120,56 @@ class FaceDetector(object):
"""
Detects the biggest face and crop it based in the eyes location
using py:class:`bob.ip.base.FaceEyesNorm`.
Final eyes location was inspired here: https://gitlab.idiap.ch/bob/bob.bio.caffe_face/blob/master/bob/bio/caffe_face/config/preprocessor/vgg_preprocessor.py
**Parameters**
**Parameters**
image: numpy array with color image [c, w, h]
final_image_size: Image dimensions [w, h]
**Returns**
The cropped image. If there's no face, `None` will be returned
"""
face, landmark = self.detect_single_face(image)
if face is None:
return None
CROPPED_IMAGE_WIDTH = final_image_size[0]
CROPPED_IMAGE_HEIGHT = final_image_size[1]
# final image position w.r.t the image size
RIGHT_EYE_POS = (CROPPED_IMAGE_HEIGHT / 3.44, CROPPED_IMAGE_WIDTH / 3.02)
LEFT_EYE_POS = (CROPPED_IMAGE_HEIGHT / 3.44 , CROPPED_IMAGE_WIDTH / 1.49)
extractor = bob.ip.base.FaceEyesNorm((CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH), RIGHT_EYE_POS, LEFT_EYE_POS)
return extractor(image, landmark['reye'], landmark['leye'])
def detect_crop(self, image, final_image_size=(182, 182), margin=44):
"""
Detects the biggest face and crop it
**Parameters**
**Parameters**
image: numpy array with color image [c, w, h]
final_image_size: Image dimensions [w, h]
**Returns**
The cropped image. If there's no face, `None` will be returned
"""
face, landmark = self.detect_single_face(image)
if face is None:
return None
top = numpy.maximum(face.top - margin/2, 0)
left = numpy.maximum(face.left - margin/2, 0)
top = numpy.uint(numpy.maximum(face.top - margin/2, 0))
left = numpy.uint(numpy.maximum(face.left - margin/2, 0))
bottom = numpy.minimum(face.bottom + margin/2, image.shape[1])
right = numpy.minimum(face.right + margin/2, image.shape[2])
bottom = numpy.uint(numpy.minimum(face.bottom + margin/2, image.shape[1]))
right = numpy.uint(numpy.minimum(face.right + margin/2, image.shape[2]))
cropped = image[:, top:bottom, left:right]
......
......@@ -8,36 +8,59 @@ import pkg_resources
detector = bob.ip.mtcnn.FaceDetector()
import numpy as np
def test_face_detection():
### Testing multiple detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.mtcnn', 'data/multiple-faces.jpg'))
faces, landmarks = detector.detect_all_faces(color_image)
assert len(faces) == 18
assert len(landmarks) == 18
### Testing multiple detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.mtcnn', 'data/multiple-faces.jpg'))
faces, landmarks = detector.detect_all_faces(color_image)
assert len(faces) == 18
assert len(landmarks) == 18
possible_landmarks = ['reye', 'leye', 'nose', 'mouthleft', 'mouthright']
for p in possible_landmarks:
assert p in landmarks[0]
possible_landmarks = ['reye', 'leye', 'nose', 'mouthleft', 'mouthright']
for p in possible_landmarks:
assert p in landmarks[0]
### Testing single detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.facedetect', 'data/testimage.jpg'))
faces, landmarks = detector.detect_single_face(color_image)
assert isinstance(faces, BoundingBox)
### Testing single detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.facedetect', 'data/testimage.jpg'))
faces, landmarks = detector.detect_single_face(color_image)
assert isinstance(faces, BoundingBox)
### Testing no detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.mtcnn', 'data/jeep.jpg'))
faces, landmarks = detector.detect_single_face(color_image)
assert faces is None
assert landmarks is None
### Testing no detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.mtcnn', 'data/jeep.jpg'))
faces, landmarks = detector.detect_single_face(color_image)
assert faces is None
assert landmarks is None
#=========================================================================
# assert negative coordinates and even arrays:
def test_crop():
image = np.zeros((3, 100, 100))
result = detector.detect_single_face(image)
assert result == (None, None)
image = np.ones((3, 100, 100))
result = detector.detect_single_face(image)
assert result == (None, None)
# test on the actual image:
test_file = pkg_resources.resource_filename('bob.ip.mtcnn', 'data/test_image.hdf5')
f = bob.io.base.HDF5File(test_file) #read only
image = f.read('image') #reads integer
del f
### Testing multiple detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.facedetect', 'data/testimage.jpg'))
face = detector.detect_crop_align(color_image, final_image_size=(224, 224))
assert face.shape == (3, 224, 224)
result = detector.detect_single_face(image)
assert result[0].topleft == (0, 77)
assert result[0].bottomright == (225, 246)
def test_crop():
### Testing multiple detections
color_image = bob.io.base.load(pkg_resources.resource_filename('bob.ip.facedetect', 'data/testimage.jpg'))
face = detector.detect_crop_align(color_image, final_image_size=(224, 224))
assert face.shape == (3, 224, 224)
......@@ -53,13 +53,14 @@ def rectangle2bounding_box2(raw_bounding_boxes):
"""
Converts a bob.ip.facedetect.BoundingBox
"""
def convert_one(bb):
assert len(bb)==5
topleft = (bb[1], bb[0])
# topleft = (bb[1], bb[0])
topleft = (numpy.max([0, bb[1]]), numpy.max([0, bb[0]]))
size = (bb[3] - bb[1], bb[2] - bb[0])
return BoundingBox(topleft, size)
# If it is only one
if raw_bounding_boxes.ndim == 1:
return convert_one(raw_bounding_boxes)
......@@ -67,7 +68,7 @@ def rectangle2bounding_box2(raw_bounding_boxes):
bounding_boxes = []
for b in raw_bounding_boxes:
bounding_boxes.append(convert_one(b))
return bounding_boxes
......
......@@ -95,7 +95,8 @@ The detection of landmarks can be done as the following:
>>> import bob.io.base
>>> import bob.io.base.test_utils
>>> color_image = bob.io.base.load(bob.io.base.test_utils.datafile('testimage.jpg', 'bob.ip.facedetect'))
>>> normimage = bob.ip.mtcnn.FaceDetector().detect_crop_align(color_image, final_image_size=(224, 224))
>>> color_image_norm = bob.ip.mtcnn.FaceDetector().detect_crop(color_image, final_image_size=(224, 224))
>>> color_image_norm_align = bob.ip.mtcnn.FaceDetector().detect_crop_align(color_image, final_image_size=(224, 224))
.. plot:: plot/plot_align_faces.py
......
......@@ -9,17 +9,22 @@ import bob.ip.draw
from bob.ip.facedetect import BoundingBox
color_image = bob.io.base.load(bob.io.base.test_utils.datafile('testimage.jpg', 'bob.ip.facedetect'))
face = bob.ip.mtcnn.FaceDetector().detect_crop_align(color_image, final_image_size=(224, 224))
face_crop = bob.ip.mtcnn.FaceDetector().detect_crop(color_image, final_image_size=(224, 224))
face_crop_aligned = bob.ip.mtcnn.FaceDetector().detect_crop_align(color_image, final_image_size=(224, 224))
ax = pyplot.subplot(1, 2, 1)
ax = pyplot.subplot(1, 3, 1)
ax.set_title("Before normalization")
pyplot.imshow(bob.ip.mtcnn.utils.bob_to_dlib_image_convertion(color_image , change_color=False))
pyplot.axis('off')
ax = pyplot.subplot(1, 2, 2)
ax = pyplot.subplot(1, 3, 2)
ax.set_title("Normalized")
pyplot.imshow(bob.ip.mtcnn.utils.bob_to_dlib_image_convertion(face, change_color=False))
pyplot.imshow(bob.ip.mtcnn.utils.bob_to_dlib_image_convertion(face_crop, change_color=False))
pyplot.axis('off')
ax = pyplot.subplot(1, 3, 3)
ax.set_title("Normalized and aligned")
pyplot.imshow(bob.ip.mtcnn.utils.bob_to_dlib_image_convertion(face_crop_aligned, change_color=False))
pyplot.axis('off')
#pyplot.show()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment