diff --git a/.gitignore b/.gitignore index 1d06de1f736efc0cda0e787d107d7faa974fd5ed..42462d96cc1f8d4ecfc6ecadb80e5acc12b46e50 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ develop-eggs sphinx dist bob/ip/tensorflow_extractor/data/FaceNet/ +record.txt +build/ +bob/ip/tensorflow_extractor/data/DR_GAN_model/ diff --git a/bob/ip/tensorflow_extractor/Extractor.py b/bob/ip/tensorflow_extractor/Extractor.py index 46ba5dea479076b5da8204d142fedc4e8ea0afdf..a3c3f6d120f4f5baa1218fcc83faee7cd12d6f94 100755 --- a/bob/ip/tensorflow_extractor/Extractor.py +++ b/bob/ip/tensorflow_extractor/Extractor.py @@ -63,12 +63,12 @@ class Extractor(object): Parameters ---------- - image : numpy.array + image : numpy.ndarray Input Data Returns ------- - numpy.array + numpy.ndarray The features. """ diff --git a/bob/ip/tensorflow_extractor/MTCNN.py b/bob/ip/tensorflow_extractor/MTCNN.py new file mode 100644 index 0000000000000000000000000000000000000000..1e21778714e17bf259c8b9180b3d5f24eec339fe --- /dev/null +++ b/bob/ip/tensorflow_extractor/MTCNN.py @@ -0,0 +1,125 @@ +# code and model from https://github.com/blaueck/tf-mtcnn +import pkg_resources +import tensorflow as tf +import multiprocessing +import bob.io.image + + +MODEL_PATH = pkg_resources.resource_filename(__name__, "data/mtcnn/mtcnn.pb") + + +class MTCNN: + + """MTCNN v1 wrapper. See + https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for more + details on MTCNN and see :ref:`bob.ip.tensorflow_extractor.face_detect` for an + example code. + + Attributes + ---------- + factor : float + Factor is a trade-off between performance and speed. + min_size : int + Minimum face size to be detected. + thresholds : list + thresholds are a trade-off between false positives and missed detections. + """ + + def __init__( + self, + min_size=40, + factor=0.709, + thresholds=(0.6, 0.7, 0.7), + model_path=MODEL_PATH, + ): + self.min_size = min_size + self.factor = factor + self.thresholds = thresholds + + graph = tf.Graph() + with graph.as_default(): + with open(model_path, "rb") as f: + graph_def = tf.GraphDef.FromString(f.read()) + tf.import_graph_def(graph_def, name="") + self.graph = graph + config = tf.ConfigProto( + intra_op_parallelism_threads=multiprocessing.cpu_count(), + inter_op_parallelism_threads=multiprocessing.cpu_count(), + ) + self.sess = tf.Session(graph=graph, config=config) + + def detect(self, img): + """Detects all faces in the image. + + Parameters + ---------- + img : numpy.ndarray + An RGB image in Bob format. + + Returns + ------- + tuple + A tuple of boxes, probabilities, and landmarks. + """ + # assuming img is Bob format and RGB + assert img.shape[0] == 3, img.shape + # network expects BGR opencv format + img = bob.io.image.to_matplotlib(img) + img = img[..., ::-1] + feeds = { + self.graph.get_operation_by_name("input").outputs[0]: img, + self.graph.get_operation_by_name("min_size").outputs[0]: self.min_size, + self.graph.get_operation_by_name("thresholds").outputs[0]: self.thresholds, + self.graph.get_operation_by_name("factor").outputs[0]: self.factor, + } + fetches = [ + self.graph.get_operation_by_name("prob").outputs[0], + self.graph.get_operation_by_name("landmarks").outputs[0], + self.graph.get_operation_by_name("box").outputs[0], + ] + prob, landmarks, box = self.sess.run(fetches, feeds) + return box, prob, landmarks + + def annotations(self, img): + """Detects all faces in the image + + Parameters + ---------- + img : numpy.ndarray + An RGB image in Bob format. + + Returns + ------- + list + A list of annotations. Annotations are dictionaries that contain the + following keys: ``topleft``, ``bottomright``, ``reye``, ``leye``, ``nose``, + ``mouthright``, ``mouthleft``, and ``quality``. + """ + boxes, scores, landmarks = self.detect(img) + annots = [] + for box, prob, lm in zip(boxes, scores, landmarks): + topleft = box[0], box[1] + bottomright = box[2], box[3] + right_eye = lm[0], lm[5] + left_eye = lm[1], lm[6] + nose = lm[2], lm[7] + mouthright = lm[3], lm[8] + mouthleft = lm[4], lm[9] + annots.append( + { + "topleft": topleft, + "bottomright": bottomright, + "reye": right_eye, + "leye": left_eye, + "nose": nose, + "mouthright": mouthright, + "mouthleft": mouthleft, + "quality": prob, + } + ) + return annots + + def __call__(self, img): + """Wrapper for the annotations method. + """ + return self.annotations(img) diff --git a/bob/ip/tensorflow_extractor/__init__.py b/bob/ip/tensorflow_extractor/__init__.py index 9e17cab7760353539a00c3302ac0ea454b4f47c1..3899dd2eb915bedb2ac434089f95d739ea96f276 100755 --- a/bob/ip/tensorflow_extractor/__init__.py +++ b/bob/ip/tensorflow_extractor/__init__.py @@ -37,6 +37,7 @@ from .Extractor import Extractor from .FaceNet import FaceNet from .DrGanMSU import DrGanMSUExtractor from .Vgg16 import VGGFace, vgg_16 +from .MTCNN import MTCNN # gets sphinx autodoc done right - don't remove it @@ -59,7 +60,8 @@ __appropriate__( Extractor, FaceNet, DrGanMSUExtractor, - VGGFace + VGGFace, + MTCNN, ) # gets sphinx autodoc done right - don't remove it diff --git a/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.hdf5 b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..bbd99614a26b381e6c7fc9876a33bdc7c9bba323 Binary files /dev/null and b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.hdf5 differ diff --git a/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.json b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.json new file mode 100644 index 0000000000000000000000000000000000000000..65f18d66f8dc57eb39ceb96f36c4773930b792cb --- /dev/null +++ b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.json @@ -0,0 +1,188 @@ +[ + { + "topleft": [ + 40.24329, + 113.20566 + ], + "bottomright": [ + 106.32423, + 162.45758 + ], + "reye": [ + 67.90105, + 124.33353 + ], + "leye": [ + 64.25375, + 145.77344 + ], + "nose": [ + 81.764984, + 136.25734 + ], + "mouthright": [ + 88.05158, + 128.52052 + ], + "mouthleft": [ + 84.64868, + 150.7494 + ], + "quality": 0.99999917 + }, + { + "topleft": [ + 34.18492, + 427.5858 + ], + "bottomright": [ + 94.81634, + 471.9476 + ], + "reye": [ + 56.84209, + 446.76434 + ], + "leye": [ + 64.74565, + 462.9847 + ], + "nose": [ + 71.690926, + 454.36282 + ], + "mouthright": [ + 75.266556, + 438.5781 + ], + "mouthleft": [ + 82.32741, + 454.99423 + ], + "quality": 0.99981314 + }, + { + "topleft": [ + 69.87796, + 31.797615 + ], + "bottomright": [ + 123.43042, + 78.04486 + ], + "reye": [ + 96.72684, + 43.90791 + ], + "leye": [ + 89.42775, + 61.957954 + ], + "nose": [ + 104.02164, + 56.380474 + ], + "mouthright": [ + 111.99057, + 49.277725 + ], + "mouthleft": [ + 104.45787, + 69.476105 + ], + "quality": 0.9994398 + }, + { + "topleft": [ + 105.99489, + 238.27567 + ], + "bottomright": [ + 159.2981, + 280.56006 + ], + "reye": [ + 125.97672, + 249.38193 + ], + "leye": [ + 127.117195, + 268.7624 + ], + "nose": [ + 138.73158, + 257.6499 + ], + "mouthright": [ + 142.67319, + 246.85234 + ], + "mouthleft": [ + 143.5627, + 267.6805 + ], + "quality": 0.99918956 + }, + { + "topleft": [ + 48.377903, + 321.892 + ], + "bottomright": [ + 110.95402, + 367.87064 + ], + "reye": [ + 73.381096, + 334.53403 + ], + "leye": [ + 76.274086, + 355.40384 + ], + "nose": [ + 86.6857, + 344.3223 + ], + "mouthright": [ + 94.80564, + 331.12646 + ], + "mouthleft": [ + 96.63391, + 351.96518 + ], + "quality": 0.9987685 + }, + { + "topleft": [ + 115.29803, + 159.48656 + ], + "bottomright": [ + 172.41876, + 205.56857 + ], + "reye": [ + 141.30688, + 171.35336 + ], + "leye": [ + 137.49718, + 191.17722 + ], + "nose": [ + 151.32994, + 182.92662 + ], + "mouthright": [ + 159.72272, + 175.2344 + ], + "mouthleft": [ + 156.25536, + 193.2938 + ], + "quality": 0.99671644 + } +] diff --git a/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.pb b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.pb new file mode 100644 index 0000000000000000000000000000000000000000..4cc80c6c9b6bd45bb6c64e030a8433bed2726b30 Binary files /dev/null and b/bob/ip/tensorflow_extractor/data/mtcnn/mtcnn.pb differ diff --git a/bob/ip/tensorflow_extractor/data/mtcnn/test_image.png b/bob/ip/tensorflow_extractor/data/mtcnn/test_image.png new file mode 100644 index 0000000000000000000000000000000000000000..f42a01af81fae4cb02e73e90a2f1cc0297b07822 Binary files /dev/null and b/bob/ip/tensorflow_extractor/data/mtcnn/test_image.png differ diff --git a/bob/ip/tensorflow_extractor/test.py b/bob/ip/tensorflow_extractor/test.py index ef80394644a694d3eb24faa687aadbc62aa7e308..dff72a5ba1faa6372228f12bdcc6454a5a395aa0 100644 --- a/bob/ip/tensorflow_extractor/test.py +++ b/bob/ip/tensorflow_extractor/test.py @@ -1,13 +1,16 @@ import bob.io.base +import bob.io.image from bob.io.base.test_utils import datafile import bob.ip.tensorflow_extractor import tensorflow as tf import pkg_resources import numpy -numpy.random.seed(10) +import json import os +numpy.random.seed(10) + slim = tf.contrib.slim from . import scratch_network @@ -15,8 +18,9 @@ from . import scratch_network def test_output(): # Loading MNIST model - filename = os.path.join(pkg_resources.resource_filename( - __name__, 'data'), 'model.ckp') + filename = os.path.join( + pkg_resources.resource_filename(__name__, "data"), "model.ckp" + ) inputs = tf.placeholder(tf.float32, shape=(None, 28, 28, 1)) # Testing the last output @@ -41,23 +45,53 @@ def test_output(): def test_facenet(): from bob.ip.tensorflow_extractor import FaceNet + extractor = FaceNet() data = numpy.random.rand(3, 160, 160).astype("uint8") output = extractor(data) assert output.size == 128, output.shape + def test_drgan(): from bob.ip.tensorflow_extractor import DrGanMSUExtractor + extractor = DrGanMSUExtractor() data = numpy.random.rand(3, 96, 96).astype("uint8") output = extractor(data) assert output.size == 320, output.shape + def test_vgg16(): pass - #from bob.ip.tensorflow_extractor import VGGFace - #extractor = VGGFace() - #data = numpy.random.rand(3, 224, 224).astype("uint8") - #output = extractor(data) - #assert output.size == 4096, output.shape + # from bob.ip.tensorflow_extractor import VGGFace + # extractor = VGGFace() + # data = numpy.random.rand(3, 224, 224).astype("uint8") + # output = extractor(data) + # assert output.size == 4096, output.shape + + +def test_mtcnn(): + test_image = datafile("mtcnn/test_image.png", __name__) + ref_numbers = datafile("mtcnn/mtcnn.hdf5", __name__) + ref_annots = datafile("mtcnn/mtcnn.json", __name__) + from bob.ip.tensorflow_extractor import MTCNN + + mtcnn = MTCNN() + img = bob.io.base.load(test_image) + bbox, prob, landmarks = mtcnn.detect(img) + with bob.io.base.HDF5File(ref_numbers, "r") as f: + ref_bbox = f["bbox"] + ref_scores = f["scores"] + ref_landmarks = f["landmarks"] + + assert numpy.allclose(bbox, ref_bbox), (bbox, ref_bbox) + assert numpy.allclose(prob, ref_scores), (prob, ref_scores) + assert numpy.allclose(landmarks, ref_landmarks), (landmarks, ref_landmarks) + annots = mtcnn.annotations(img) + ref_annots = json.load(open(ref_annots)) + assert len(annots) == len(ref_annots), (len(annots), len(ref_annots)) + for a, aref in zip(annots, ref_annots): + for k, v in a.items(): + vref = aref[k] + assert numpy.allclose(v, vref) diff --git a/conda/meta.yaml b/conda/meta.yaml index c845a4c370911c99f5c9cd7250ba4db8c83f6af0..f4d97e9f0407079f3652d5914c4b871f9a1c06a1 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -25,12 +25,14 @@ requirements: - bob.db.mnist - bob.ip.color - six {{ six }} + - tensorflow {{ tensorflow }} + - numpy {{ numpy }} run: - python - setuptools - - scipy - six - - tensorflow >=1.2.1 + - {{ pin_compatible('tensorflow') }} + - {{ pin_compatible('numpy') }} test: imports: @@ -47,7 +49,6 @@ test: - coverage - sphinx - sphinx_rtd_theme - - bob.io.image - bob.db.atnt - matplotlib - gridtk diff --git a/doc/conf.py b/doc/conf.py index 3e3953b58dabb0fd886dc049e628d5756167c9cc..bb602aa67dd0932d115cd02503ac188f78ff4b2d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -36,7 +36,7 @@ else: extensions.append('sphinx.ext.pngmath') # Be picky about warnings -nitpicky = False +nitpicky = True # Ignores stuff we can't easily resolve on other project's sphinx manuals nitpick_ignore = [] diff --git a/doc/face_detect.rst b/doc/face_detect.rst new file mode 100644 index 0000000000000000000000000000000000000000..1ea59fcdeef2ecd65db95a150e685050abc389fb --- /dev/null +++ b/doc/face_detect.rst @@ -0,0 +1,18 @@ + +.. _bob.ip.tensorflow_extractor.face_detect: + +============================ + Face detection using MTCNN +============================ + +This package comes with a wrapper around the MTCNN (v1) face detector. See +https://kpzhang93.github.io/MTCNN_face_detection_alignment/index.html for more +information on MTCNN. The model is directly converted from the caffe model using code in +https://github.com/blaueck/tf-mtcnn + +See below for an example on how to use +:any:`bob.ip.tensorflow_extractor.MTCNN`: + +.. plot:: plot/detect_faces_mtcnn.py + :include-source: True + diff --git a/doc/index.rst b/doc/index.rst index 8858af850cff2648d15c1a7fba82c90cac15fadc..cc4daeda6d67c165b0ec5bb3f64dc49dbc8ae15d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,4 +16,5 @@ Tensorflow http://tensorflow.org/ :maxdepth: 2 guide + face_detect py_api diff --git a/doc/plot/detect_faces_mtcnn.py b/doc/plot/detect_faces_mtcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..570f6b7c43657c0b52b8eb023ec5fc023734b119 --- /dev/null +++ b/doc/plot/detect_faces_mtcnn.py @@ -0,0 +1,53 @@ +from bob.io.image import imshow +from bob.io.base import load +from bob.io.base.test_utils import datafile +from bob.ip.tensorflow_extractor import MTCNN +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle, Circle + +# load colored test image +color_image = load(datafile("mtcnn/test_image.png", "bob.ip.tensorflow_extractor")) + +# detect all face +detector = MTCNN() +detections = detector(color_image) + +imshow(color_image) +plt.axis("off") + +for annotations in detections: + topleft = annotations["topleft"] + bottomright = annotations["bottomright"] + size = bottomright[0] - topleft[0], bottomright[1] - topleft[1] + # draw bounding boxes + plt.gca().add_patch( + Rectangle( + topleft[::-1], + size[1], + size[0], + edgecolor="b", + facecolor="none", + linewidth=2, + ) + ) + # draw face landmarks + for key, color in ( + ("reye", "r"), + ("leye", "g"), + ("nose", "b"), + ("mouthright", "k"), + ("mouthleft", "w"), + ): + plt.gca().add_patch( + Circle( + annotations[key][::-1], + radius=2, + edgecolor=color, + facecolor="none", + linewidth=2, + ) + ) + # show quality of detections + plt.text( + topleft[1], topleft[0], round(annotations["quality"], 3), color="b", fontsize=14 + ) diff --git a/doc/py_api.rst b/doc/py_api.rst index a4bd582d6d5a8f617d76b7b95824ca1dfa4f952d..454ee6f5c43f1831e4b38684f1afcd151fa556d1 100644 --- a/doc/py_api.rst +++ b/doc/py_api.rst @@ -9,6 +9,7 @@ Classes .. autosummary:: bob.ip.tensorflow_extractor.Extractor bob.ip.tensorflow_extractor.FaceNet + bob.ip.tensorflow_extractor.MTCNN Detailed API