Improve tests to use OpenCV python bindings if available; Test if keypoints...

Improve tests to use OpenCV python bindings if available; Test if keypoints are inside bounding-boxes

xbob/ip/flandmark/flandmark.cpp

@@ -49,7 +49,6 @@ static auto s_class = xbob::extension::ClassDoc(
 typedef struct {
   PyObject_HEAD
   FLANDMARK_Model* flandmark;
-  double* landmarks;
   std::string filename;
 } PyBobIpFlandmarkObject;
 
@@ -97,25 +96,15 @@ static int PyBobIpFlandmark_init
     return -1;
   }
 
-  //flandmark is now initialized, allocate keypoint place-holder
-  self->landmarks = new double[2*self->flandmark->data.options.M];
-  if (!self->landmarks) {
-    flandmark_free(self->flandmark);
-    PyErr_Format(PyExc_RuntimeError, "`%s' could not allocate memory for %d keypoint place-holders for model `%s'", Py_TYPE(self)->tp_name, 2*self->flandmark->data.options.M, c_filename);
-    return -1;
-  }
-
-  //set filename
+  //flandmark is now initialized, set filename
   self->filename = c_filename;
 
-  //all good, flandmark is initialized
+  //all good, flandmark is ready
   return 0;
 
 }
 
 static void PyBobIpFlandmark_delete (PyBobIpFlandmarkObject* self) {
-  delete[] self->landmarks;
-  self->landmarks = 0;
   flandmark_free(self->flandmark);
   self->flandmark = 0;
   Py_TYPE(self)->tp_free((PyObject*)self);
@@ -139,32 +128,35 @@ static PyObject* call(PyBobIpFlandmarkObject* self,
 
   for (int i=0; i<nbbx; ++i) {
 
+    //allocate output array _and_ Flandmark buffer within a single structure
+    Py_ssize_t shape[2];
+    shape[0] = self->flandmark->data.options.M;
+    shape[1] = 2;
+    PyObject* landmarks = PyArray_SimpleNew(2, shape, NPY_FLOAT64);
+    if (!landmarks) return 0;
+    auto landmarks_ = make_safe(landmarks);
+    double* buffer = reinterpret_cast<double*>(PyArray_DATA((PyArrayObject*)landmarks));
+
     int result = 0;
     Py_BEGIN_ALLOW_THREADS
-    result = flandmark_detect(image.get(), &bbx[4*i],
-        self->flandmark, self->landmarks);
+    result = flandmark_detect(image.get(), &bbx[4*i], self->flandmark, buffer);
     Py_END_ALLOW_THREADS
 
-    PyObject* landmarks = 0;
     if (result != NO_ERR) {
       Py_INCREF(Py_None);
       landmarks = Py_None;
     }
     else {
-      landmarks = PyTuple_New(self->flandmark->data.options.M);
-      if (!landmarks) return 0;
-      auto landmarks_ = make_safe(landmarks);
+      //swap keypoint coordinates (x, y) -> (y, x)
+      double tmp;
       for (int k = 0; k < (2*self->flandmark->data.options.M); k += 2) {
-        PyTuple_SET_ITEM(landmarks, k/2,
-            Py_BuildValue("dd",
-              self->landmarks[k+1], //y value
-              self->landmarks[k]    //x value
-              )
-            );
-        if (!PyTuple_GET_ITEM(landmarks, k/2)) return 0;
+        tmp         = buffer[k];
+        buffer[k]   = buffer[k+1];
+        buffer[k+1] = tmp;
       }
       Py_INCREF(landmarks);
     }
+
     PyTuple_SET_ITEM(retval, i, landmarks);
 
   }
@@ -192,7 +184,7 @@ static auto s_call = xbob::extension::FunctionDoc(
     "4. Mouth-corner-l (left corner of the mouth)\n"
     "5. Canthus-rr (outer corner of the right eye)\n"
     "6. Canthus-ll (outer corner of the left eye)\n"
-    "7.  Nose\n"
+    "7. Nose\n"
     "\n"
     "Each point is returned as tuple defining the pixel positions in the form "
     "(y, x).\n"
@@ -203,7 +195,7 @@ static auto s_call = xbob::extension::FunctionDoc(
       "The image Flandmark will operate on")
     .add_parameter("y, x", "int", "The top left-most corner of the bounding box containing the face image you want to locate keypoints on.")
     .add_parameter("height, width", "int", "The dimensions accross ``y`` (height) and ``x`` (width) for the bounding box, in number of pixels.")
-    .add_return("landmarks", "tuple", "A sequence of tuples, each containing locations in the format ``(y, x)``, for each of the key-points defined above and in that order.")
+    .add_return("landmarks", "array (2D, float64)", "Each row in the output array contains the locations of keypoints in the format ``(y, x)``")
     ;
 
 static PyObject* PyBobIpFlandmark_call_single(PyBobIpFlandmarkObject* self,

xbob/ip/flandmark/test.py

@@ -7,6 +7,8 @@
 """
 
 import os
+import numpy
+import functools
 import pkg_resources
 import nose.tools
 import xbob.io
@@ -20,7 +22,7 @@ def F(f):
 
 LENA = F('lena.jpg')
 LENA_BBX = [
-    (214, 202, 183, 183)
+    [214, 202, 183, 183]
     ] #from OpenCV's cascade detector
 
 MULTI = F('multi.jpg')
@@ -45,10 +47,75 @@ def opencv_detect(image):
       1.3, #scaleFactor (at each time the image is re-scaled)
       4, #minNeighbors (around candidate to be retained)
       0, #flags (normally, should be set to zero)
-      (20,20), #(minSize, maxSize) (of detected objects)
+      (20,20), #(minSize, maxSize) (of detected objects on that scale)
       )
 
-@nose.tools.nottest
+def pnpoly(point, vertices):
+  """Python translation of the C algorithm taken from:
+  http://www.ecse.rpi.edu/Homepages/wrf/Research/Short_Notes/pnpoly.html
+  """
+
+  (x, y) = point
+  j = vertices[-1]
+  c = False
+  for i in vertices:
+    if ( (i[1] > y) != (j[1] > y) ) and \
+        ( x < (((j[0]-i[0]) * (y-i[1]) / (j[1]-i[1])) + i[0]) ):
+      c = not c
+    j = i
+
+  return c
+
+def is_inside(point, box, eps=1e-5):
+  """Calculates, using matplotlib, if a point lies inside a bounding box"""
+
+  (y, x, height, width) = box
+  #note: vertices must be organized clockwise
+  vertices = numpy.array([
+    (x-eps, y-eps),
+    (x+width+eps, y-eps),
+    (x+width+eps, y+height+eps),
+    (x-eps, y+height+eps),
+    ], dtype=float)
+  return pnpoly((point[1], point[0]), vertices)
+
+def opencv_available(test):
+  """Decorator for detecting if OpenCV/Python bindings are available"""
+
+  @functools.wraps(test)
+  def wrapper(*args, **kwargs):
+    try:
+      import cv2
+      return test(*args, **kwargs)
+    except ImportError:
+      raise SkipTest("The cv2 module is not available")
+
+  return wrapper
+
+def test_is_inside():
+
+  box = (0, 0, 1, 1)
+
+  # really inside
+  assert is_inside((0.5, 0.5), box, eps=1e-10)
+
+  # on the limit of the box
+  assert is_inside((0.0, 0.0), box, eps=1e-10)
+  assert is_inside((1.0, 1.0), box, eps=1e-10)
+  assert is_inside((1.0, 0.0), box, eps=1e-10)
+  assert is_inside((0.0, 1.0), box, eps=1e-10)
+
+def test_is_outside():
+
+  box = (0, 0, 1, 1)
+
+  # really outside the box
+  assert not is_inside((1.5, 1.0), box, eps=1e-10)
+  assert not is_inside((0.5, 1.5), box, eps=1e-10)
+  assert not is_inside((1.5, 1.5), box, eps=1e-10)
+  assert not is_inside((-0.5, -0.5), box, eps=1e-10)
+
+@opencv_available
 def test_lena_opencv():
 
   img = xbob.io.load(LENA)
@@ -57,7 +124,10 @@ def test_lena_opencv():
 
   flm = Flandmark()
   keypoints = flm.locate(gray, y, x, height, width)
-  assert keypoints
+  nose.tools.eq_(keypoints.shape, (8, 2))
+  nose.tools.eq_(keypoints.dtype, 'float64')
+  for k in keypoints:
+    assert is_inside(k, (y, x, height, width), eps=1)
 
 def test_lena():
 
@@ -67,10 +137,12 @@ def test_lena():
 
   flm = Flandmark()
   keypoints = flm.locate(gray, y, x, height, width)
-  assert keypoints
-  nose.tools.eq_(len(keypoints), 8)
+  nose.tools.eq_(keypoints.shape, (8, 2))
+  nose.tools.eq_(keypoints.dtype, 'float64')
+  for k in keypoints:
+    assert is_inside(k, (y, x, height, width), eps=1)
 
-@nose.tools.nottest
+@opencv_available
 def test_multi_opencv():
 
   img = xbob.io.load(MULTI)
@@ -80,7 +152,10 @@ def test_multi_opencv():
   flm = Flandmark()
   for (x, y, width, height) in bbx:
     keypoints = flm.locate(gray, y, x, height, width)
-    assert keypoints
+    nose.tools.eq_(keypoints.shape, (8, 2))
+    nose.tools.eq_(keypoints.dtype, 'float64')
+    for k in keypoints:
+      assert is_inside(k, (y, x, height, width), eps=1)
 
 def test_multi():
 
@@ -90,5 +165,7 @@ def test_multi():
   flm = Flandmark()
   for (x, y, width, height) in MULTI_BBX:
     keypoints = flm.locate(gray, y, x, height, width)
-    assert keypoints
-    nose.tools.eq_(len(keypoints), 8)
+    nose.tools.eq_(keypoints.shape, (8, 2))
+    nose.tools.eq_(keypoints.dtype, 'float64')
+    for k in keypoints:
+      assert is_inside(k, (y, x, height, width), eps=1)