Merge branch 'celeba_preproc' into 'master'

Preprocessing and quality check

See merge request !83

.gitignore

@@ -14,5 +14,4 @@ dist
 record.txt
 results
 submitted.sql3
-config
 temp*

bob/pad/face/config/preprocessor/face_feature_crop_quality_check.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Oct  9 13:53:58 2018
+
+@author: Olegs Nikisins
+"""
+
+# =============================================================================
+# Import here:
+
+from bob.bio.base.preprocessor import Preprocessor
+
+from bob.bio.video.preprocessor import Wrapper
+
+import os
+
+import importlib
+
+from bob.pad.face.utils.patch_utils import reshape_flat_patches
+
+from bob.bio.video.utils import FrameSelector
+
+from bob.pad.face.preprocessor import BlockPatch
+
+
+# =============================================================================
+# define preprocessor class:
+
+class _Preprocessor(Preprocessor):
+    """
+    The following steps are performed:
+
+    1. Detect and align the face.
+
+    2. Assess the quality of the face image.
+
+    3. Extract patch / patches from the face.
+
+    **Parameters:**
+
+    ``face_crop_align`` : object
+        An instance of the FaceCropAlign preprocessor to be used in step one.
+
+    ``config_file``: py:class:`string`
+        Relative name of the config file containing
+        quality assessment function.
+        Example: ``celeb_a/quality_assessment_config.py``.
+
+    ``config_group``: py:class:`string`
+        Group/package name containing the configuration file.
+        Example: ``bob.pad.face.config.quality_assessment``.
+
+    ``block_patch`` : object
+        An instance of the BlockPatch preprocessor to be used in step 3.
+
+    ``patch_reshape_parameters`` : [int] or None
+        The parameters to be used for patch reshaping. The patch is
+        vectorized. Example:
+        ``patch_reshape_parameters = [4, 8, 8]``, then the patch of the
+        size (256,) will be reshaped to (4,8,8) dimensions. Only 2D and 3D
+        patches are supported.
+        Default: None.
+
+    ``patch_num`` : int OR None
+        Am index of the patch to be selected from all extracted patches.
+        Default: None
+    """
+
+    def __init__(self,
+                 face_crop_align,
+                 config_file,
+                 config_group,
+                 block_patch,
+                 patch_reshape_parameters = None,
+                 patch_num = None):
+
+        super(_Preprocessor, self).__init__()
+
+        self.face_crop_align = face_crop_align
+        self.config_file = config_file
+        self.config_group = config_group
+        self.block_patch = block_patch
+        self.patch_reshape_parameters = patch_reshape_parameters
+        self.patch_num = patch_num
+
+
+    def __call__(self, data, annotations):
+        """
+        **Parameters:**
+
+        ``data`` : 2D or 3D :py:class:`numpy.ndarray`
+            Input image (RGB or gray-scale) or None.
+
+        ``annotations`` : :py:class:`dict` or None
+            A dictionary containing annotations of the face bounding box.
+            Dictionary must be as follows:
+            ``{'topleft': (row, col), 'bottomright': (row, col)}``
+            Default: None .
+        """
+
+        face_data = self.face_crop_align(data, annotations)
+
+        if face_data is None:
+
+            return None
+
+        relative_mod_name = '.' + os.path.splitext(self.config_file)[0].replace(os.path.sep, '.')
+
+        config_module = importlib.import_module(relative_mod_name, self.config_group)
+
+        quality_flag = config_module.assess_quality(face_data, **config_module.assess_quality_kwargs)
+
+        if quality_flag:
+
+            print ("Good quality data.")
+
+            patches = self.block_patch(face_data, annotations=None)
+
+            if self.patch_reshape_parameters is not None:
+
+                patches = reshape_flat_patches(patches, self.patch_reshape_parameters)
+
+            if self.patch_num is not None:
+
+                patches = patches[self.patch_num]
+
+        else:
+
+            print ("Bad quality data.")
+            return None
+
+        return patches
+
+
+# =============================================================================
+# define instance of the preprocessor:
+
+"""
+Preprocessor to be used for Color channel.
+"""
+
+from bob.pad.face.preprocessor import FaceCropAlign
+
+FACE_SIZE = 128  # The size of the resulting face
+RGB_OUTPUT_FLAG = True  # RGB output
+USE_FACE_ALIGNMENT = True  # use annotations
+MAX_IMAGE_SIZE = 1920  # no limiting here
+FACE_DETECTION_METHOD = "mtcnn"  # DON'T use ANNOTATIONS, valid for CelebA only
+MIN_FACE_SIZE = 50  # skip small faces
+
+_face_crop_align = FaceCropAlign(face_size = FACE_SIZE,
+                                 rgb_output_flag = RGB_OUTPUT_FLAG,
+                                 use_face_alignment = USE_FACE_ALIGNMENT,
+                                 max_image_size = MAX_IMAGE_SIZE,
+                                 face_detection_method = FACE_DETECTION_METHOD,
+                                 min_face_size = MIN_FACE_SIZE)
+
+"""
+Parameters to be used for quality assessment.
+"""
+
+CONFIG_FILE = "celeb_a/quality_assessment_config_128.py"
+
+CONFIG_GROUP = "bob.pad.face.config.quality_assessment"
+
+"""
+Define an instance of the BlockPatch preprocessor.
+"""
+
+PATCH_SIZE = 64
+STEP = 32
+
+_block_patch = BlockPatch(patch_size = PATCH_SIZE,
+                          step = STEP,
+                          use_annotations_flag = False)
+
+"""
+define an instance of the _Preprocessor class.
+"""
+
+_frame_selector = FrameSelector(selection_style = "all")
+
+_image_extractor_0 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 0)
+
+face_feature_0_crop_rgb = Wrapper(preprocessor = _image_extractor_0,
+                                  frame_selector = _frame_selector)
+
+
+# =============================================================================
+_image_extractor_1 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 1)
+
+face_feature_1_crop_rgb = Wrapper(preprocessor = _image_extractor_1,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_2 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 2)
+
+face_feature_2_crop_rgb = Wrapper(preprocessor = _image_extractor_2,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_3 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 3)
+
+face_feature_3_crop_rgb = Wrapper(preprocessor = _image_extractor_3,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_4 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 4)
+
+face_feature_4_crop_rgb = Wrapper(preprocessor = _image_extractor_4,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_5 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 5)
+
+face_feature_5_crop_rgb = Wrapper(preprocessor = _image_extractor_5,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_6 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 6)
+
+face_feature_6_crop_rgb = Wrapper(preprocessor = _image_extractor_6,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_7 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 7)
+
+face_feature_7_crop_rgb = Wrapper(preprocessor = _image_extractor_7,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+_image_extractor_8 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 8)
+
+face_feature_8_crop_rgb = Wrapper(preprocessor = _image_extractor_8,
+                                  frame_selector = _frame_selector)
+
+# =============================================================================
+# Extractors for obtaining RGB patches of the size 3x32x32
+
+PATCH_SIZE = 32
+STEP = 32
+
+_block_patch_32x32 = BlockPatch(patch_size = PATCH_SIZE,
+                          step = STEP,
+                          use_annotations_flag = False)
+
+_image_extractor_0_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 0)
+
+face_feature_0_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_0_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_1_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 1)
+
+face_feature_1_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_1_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_2_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 2)
+
+face_feature_2_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_2_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_3_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 3)
+
+face_feature_3_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_3_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_4_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 4)
+
+face_feature_4_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_4_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_5_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 5)
+
+face_feature_5_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_5_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_6_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 6)
+
+face_feature_6_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_6_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_7_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 7)
+
+face_feature_7_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_7_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_8_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 8)
+
+face_feature_8_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_8_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_9_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 9)
+
+face_feature_9_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_9_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_10_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 10)
+
+face_feature_10_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_10_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_11_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 11)
+
+face_feature_11_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_11_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_12_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 12)
+
+face_feature_12_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_12_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_13_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 13)
+
+face_feature_13_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_13_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_14_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 14)
+
+face_feature_14_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_14_32x32,
+                                  frame_selector = _frame_selector)
+
+
+_image_extractor_15_32x32 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_32x32,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 15)
+
+face_feature_15_32x32_crop_rgb = Wrapper(preprocessor = _image_extractor_15_32x32,
+                                  frame_selector = _frame_selector)
+
+
+# =============================================================================
+# Extractors for obtaining RGB patches (patch is an entire face in this case) of the size 3x128x128
+
+PATCH_SIZE = 128
+STEP = 1
+
+_block_patch_128x128 = BlockPatch(patch_size = PATCH_SIZE,
+                          step = STEP,
+                          use_annotations_flag = False)
+
+_image_extractor_0_128x128 = _Preprocessor(face_crop_align = _face_crop_align,
+                                   config_file = CONFIG_FILE,
+                                   config_group = CONFIG_GROUP,
+                                   block_patch = _block_patch_128x128,
+                                   patch_reshape_parameters = [3, PATCH_SIZE, PATCH_SIZE],
+                                   patch_num = 0)
+
+face_feature_0_128x128_crop_rgb = Wrapper(preprocessor = _image_extractor_0_128x128,
+                                  frame_selector = _frame_selector)
+
+
+

bob/pad/face/config/quality_assessment/celeb_a/quality_assessment_config.py

@@ -33,57 +33,7 @@ from bob.bio.video.preprocessor import Wrapper
 
 import numpy as np
 
-# =============================================================================
-def detect_eyes_in_bw_image(image):
-    """
-    Detect eyes in the image using OpenCV.
-
-    **Parameters:**
-
-    ``image`` : 2D :py:class:`numpy.ndarray`
-        A BW image to detect the eyes in.
-
-    **Returns:**
-
-    ``eyes`` : 2D :py:class:`numpy.ndarray`
-        An array containing coordinates of the bounding boxes of detected eyes.
-        The dimensionality of the array:
-        ``num_of_detected_eyes x coordinates_of_bbx``
-    """
-
-    eye_model = pkg_resources.resource_filename('bob.pad.face.config',
-                                                'quality_assessment/models/eye_detector.xml')
-
-    eye_cascade = cv2.CascadeClassifier(eye_model)
-
-    eyes = eye_cascade.detectMultiScale(image)
-
-    return eyes
-
-
-# =============================================================================
-def load_datafile(file_name):
-    """
-    Load data from file given filename. Here the data file is an hdf5 file
-    containing a framecontainer with one frame. The data in the frame is
-    a BW image of the facial region.
-
-    **Parameters:**
-
-    ``file_name`` : str
-        Absolute name of the file.
-
-    **Returns:**
-
-    ``data`` : 2D :py:class:`numpy.ndarray`
-        Data array containing the image of the facial region.
-    """
-
-    frame_container = Wrapper().read_data(file_name)
-
-    data = frame_container[0][1]
-
-    return data
+from bob.pad.face.config.quality_assessment.celeb_a.quality_assessment_config_128 import detect_eyes_in_bw_image, load_datafile, assess_quality
 
 
 # =============================================================================
@@ -98,59 +48,3 @@ assess_quality_kwargs = {}
 assess_quality_kwargs["eyes_expected"] = eyes_expected
 assess_quality_kwargs["threshold"] = 7
 
-
-# =============================================================================
-def assess_quality(data, eyes_expected, threshold):
-    """
-    Assess the quality of the data sample, which in this case is an image of
-    the face of the size 64x64 pixels. The quality assessment is based on the
-    eye detection. If two eyes are detected, and they are located in the
-    pre-defined positions, then quality is good, otherwise the quality is low.
-
-    **Parameters:**
-
-    ``data`` : 2D :py:class:`numpy.ndarray`
-        Data array containing the image of the facial region. The size of the
-        image is 64x64.
-
-    ``eyes_expected`` : list
-        A list containing expected coordinates of the eyes. The format is
-        as follows:
-        [ [left_y, left_x], [right_y, right_x] ]
-
-    ``threshold`` : int
-        A maximum allowed distance between expected and detected centers of
-        the eyes.
-
-    **Returns:**
-
-    ``quality_flag`` : bool
-        ``True`` for good quality data, ``False`` otherwise.
-    """
-
-    quality_flag = False
-
-    eyes = detect_eyes_in_bw_image(data)
-
-    if isinstance(eyes, np.ndarray):
-
-        if eyes.shape[0] == 2: # only consider the images with two eyes detected
-
-            # coordinates of detected centers of the eyes: [ [left_y, left_x], [right_y, right_x] ]:
-            eyes_detected = []
-            for (ex,ey,ew,eh) in eyes:
-                eyes_detected.append( [ey + eh/2., ex + ew/2.] )
-
-            dists = [] # dits between detected and expected:
-            for a, b in zip(eyes_detected, eyes_expected):
-                dists.append( np.linalg.norm(np.array(a)-np.array(b)) )
-
-            max_dist = np.max(dists)
-
-            if max_dist < threshold:
-
-                quality_flag = True
-
-    return quality_flag
-
-

bob/pad/face/config/quality_assessment/celeb_a/quality_assessment_config_128.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Quality assessment configuration file for the CelebA database to be used
+with quality assessment script.
+
+Note: this config checks the quality of the preprocessed(!) data. Here the
+preprocessed data is sored in ``.hdf5`` files, as a frame container with
+one frame. Frame contains a BW image of the facial regions of the size
+128x128 pixels.
+
+The config file MUST contain at least the following functions:
+
+``load_datafile(file_name)`` - returns the ``data`` given ``file_name``, and
+
+``assess_quality(data, **assess_quality_kwargs)`` - returns ``True`` for good
+quality ``data``, and ``False`` for low quality data, and
+
+``assess_quality_kwargs`` - a dictionary with kwargs for ``assess_quality()``
+function.
+
+@author: Olegs Nikisins
+"""
+
+# =============================================================================
+# Import here:
+
+import pkg_resources
+
+import cv2
+
+from bob.bio.video.preprocessor import Wrapper
+
+import numpy as np
+
+import bob.ip.color
+
+# =============================================================================
+def detect_eyes_in_bw_image(image):
+    """
+    Detect eyes in the image using OpenCV.
+
+    **Parameters:**
+
+    ``image`` : 2D :py:class:`numpy.ndarray`
+        A BW image to detect the eyes in.
+
+    **Returns:**
+
+    ``eyes`` : 2D :py:class:`numpy.ndarray`
+        An array containing coordinates of the bounding boxes of detected eyes.
+        The dimensionality of the array:
+        ``num_of_detected_eyes x coordinates_of_bbx``
+    """
+
+    eye_model = pkg_resources.resource_filename('bob.pad.face.config',
+                                                'quality_assessment/models/eye_detector.xml')
+
+    eye_cascade = cv2.CascadeClassifier(eye_model)
+
+    if len(image.shape) == 3:
+
+        image = bob.ip.color.rgb_to_gray(image)
+
+    eyes = eye_cascade.detectMultiScale(image)
+
+    return eyes
+
+
+# =============================================================================
+def load_datafile(file_name):
+    """
+    Load data from file given filename. Here the data file is an hdf5 file
+    containing a framecontainer with one frame. The data in the frame is
+    a BW image of the facial region.
+
+    **Parameters:**
+
+    ``file_name`` : str
+        Absolute name of the file.
+
+    **Returns:**
+
+    ``data`` : 2D :py:class:`numpy.ndarray`
+        Data array containing the image of the facial region.
+    """
+
+    frame_container = Wrapper().read_data(file_name)
+
+    data = frame_container[0][1]
+
+    return data
+
+
+# =============================================================================
+face_size = 128
+eyes_distance=((face_size + 1) / 2.)
+eyes_center=(face_size / 4., (face_size - 0.5) / 2.)
+
+eyes_expected = [[eyes_center[0], eyes_center[1]-eyes_distance/2.],
+                 [eyes_center[0], eyes_center[1]+eyes_distance/2.]]
+
+assess_quality_kwargs = {}
+assess_quality_kwargs["eyes_expected"] = eyes_expected
+assess_quality_kwargs["threshold"] = 10
+
+
+# =============================================================================
+def assess_quality(data, eyes_expected, threshold):
+    """
+    Assess the quality of the data sample, which in this case is an image of
+    the face of the size (face_size x face_size) pixels. The quality assessment is based on the
+    eye detection. If two eyes are detected, and they are located in the
+    pre-defined positions, then quality is good, otherwise the quality is low.
+
+    **Parameters:**
+
+    ``data`` : 2D :py:class:`numpy.ndarray`
+        Data array containing the image of the facial region. The size of the
+        image is (face_size x face_size).
+
+    ``eyes_expected`` : list
+        A list containing expected coordinates of the eyes. The format is
+        as follows:
+        [ [left_y, left_x], [right_y, right_x] ]
+
+    ``threshold`` : int
+        A maximum allowed distance between expected and detected centers of
+        the eyes.
+
+    **Returns:**
+
+    ``quality_flag`` : bool
+        ``True`` for good quality data, ``False`` otherwise.
+    """
+
+    quality_flag = False
+
+    eyes = detect_eyes_in_bw_image(data)
+
+    if isinstance(eyes, np.ndarray):
+
+        if eyes.shape[0] == 2: # only consider the images with two eyes detected
+
+            # coordinates of detected centers of the eyes: [ [left_y, left_x], [right_y, right_x] ]:
+            eyes_detected = []
+            for (ex,ey,ew,eh) in eyes:
+                eyes_detected.append( [ey + eh/2., ex + ew/2.] )
+
+            dists = [] # dits between detected and expected:
+            for a, b in zip(eyes_detected, eyes_expected):
+                dists.append( np.linalg.norm(np.array(a)-np.array(b)) )
+
+            max_dist = np.max(dists)
+
+            if max_dist < threshold:
+
+                quality_flag = True
+
+    return quality_flag
+
+

bob/pad/face/test/test.py

@@ -48,6 +48,10 @@ from bob.bio.video.utils import FrameSelector
 
 from ..preprocessor import BlockPatch
 
+from bob.pad.face.config.preprocessor.face_feature_crop_quality_check import face_feature_0_128x128_crop_rgb
+
+from bob.pad.face.utils.patch_utils import reshape_flat_patches
+
 
 def test_detect_face_landmarks_in_image_mtcnn():
 
@@ -309,6 +313,56 @@ def test_video_face_crop_align_block_patch():
     assert data_preprocessed[1][1].shape == (9, 12288)
 
 
+# =============================================================================
+def test_preproc_with_quality_check():
+    """
+    Test _Preprocessor cropping the face and checking the quality of the image
+    applying eye detection, and asserting if they are in the expected positions.
+    """
+
+    # =========================================================================
+    # prepare the test data:
+    image = load(datafile('test_image.png', 'bob.pad.face.test'))
+
+    annotations = None
+
+    video, annotations = convert_image_to_video_data(image, annotations, 2)
+
+    # =========================================================================
+    # test the preprocessor:
+    data_preprocessed = face_feature_0_128x128_crop_rgb(video)
+
+    assert data_preprocessed is None
+
+
+# =============================================================================
+def test_reshape_flat_patches():
+    """
+    Test reshape_flat_patches function.
+    """
+
+    image = load(datafile('test_image.png', 'bob.pad.face.test'))
+
+    patch1 = image[0,0:10,0:10]
+    patch2 = image[1,0:10,0:10]
+
+    patches = np.stack([patch1.flatten(), patch2.flatten()])
+    patches_3d = reshape_flat_patches(patches, (10, 10))
+
+    assert np.all(patch1 == patches_3d[0])
+    assert np.all(patch2 == patches_3d[1])
+
+    # =========================================================================
+    patch1 = image[:,0:10,0:10]
+    patch2 = image[:,1:11,1:11]
+
+    patches = np.stack([patch1.flatten(), patch2.flatten()])
+    patches_3d = reshape_flat_patches(patches, (3, 10, 10))
+
+    assert np.all(patch1 == patches_3d[0])
+    assert np.all(patch2 == patches_3d[1])
+
+
 #==============================================================================
 def test_frame_difference():
     """

bob/pad/face/utils/patch_utils.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 18 17:39:42 2019
+
+@author: Olegs Nikisins
+"""
+
+# =============================================================================
+# Import what is needed here:
+
+import numpy as np
+
+
+# =============================================================================
+# Main body:
+
+def reshape_flat_patches(patches, patch_reshape_parameters = None):
+    """
+    Reshape a set of flattened patches into original dimensions, 2D or 3D
+
+    **Parameters:**
+
+    ``patches`` : 2D :py:class:`numpy.ndarray`
+        An array containing flattened patches. The dimensions are:
+        ``num_patches x len_of_flat_patch``
+
+    ``patch_reshape_parameters`` : [int] or None
+        The parameters to be used for patch reshaping. The loaded patch is
+        vectorized. Example:
+        ``patch_reshape_parameters = [4, 8, 8]``, then the patch of the
+        size (256,) will be reshaped to (4,8,8) dimensions. Only 2D and 3D
+        patches are supported.
+        Default: None.
+
+    **Returns:**
+
+    ``patches_3d`` : [2D or 3D :py:class:`numpy.ndarray`]
+        A list of patches converted to the original dimensions.
+    """
+
+    patches_3d = []
+
+    for patch in patches:
+
+        if patch_reshape_parameters is not None:
+
+            # The dimensionality of the reshaped patch:
+            new_shape = [np.int(len(patch)/(patch_reshape_parameters[-2]*patch_reshape_parameters[-1]))] + list(patch_reshape_parameters[-2:])
+
+            patch = np.squeeze(patch.reshape(new_shape))
+
+        patches_3d.append(patch)
+
+    return patches_3d
+
+
+# =============================================================================
+def mean_std_patch_norm(patches, channel_means = None, channel_stds = None):
+    """
+    Apply mean-std normalization to the patches channel-wise.
+
+    **Parameters:**
+
+    ``patches`` : [2D or 3D :py:class:`numpy.ndarray`]
+        A list of patches converted to the original dimensions.
+
+    ``channel_means`` : [float] or None
+        The channel-wise mean values to be used for mean-std normalization
+        of the patches. Only normalization of 3D patches is currently
+        supported.
+        Default: None.
+
+    ``channel_stds`` : [float] or None
+        The channel-wise std values to be used for mean-std normalization
+        of the patches. Only normalization of 3D patches is currently
+        supported.
+        Default: None.
+
+    **Returns:**
+
+    ``patches_norm_3d`` : [2D or 3D :py:class:`numpy.ndarray`]
+        A list of patches normalized channel-wise.
+    """
+
+    patches_norm_3d = []
+
+    for patch in patches:
+
+        if channel_means is not None: # if normalization parameters are given
+
+            patch = patch.astype(np.float) # convert to float for normalization
+
+            if len(patch.shape) == 3: # Only normalization of 3D patches is currently handled
+
+                for idx, patch_channel in enumerate(patch): # for all channels
+
+                    patch[idx,:,:] = (patch_channel - channel_means[idx]) / channel_stds[idx]
+
+        patches_norm_3d.append(patch)
+
+    return patches_norm_3d
+
+

doc/index.rst

@@ -23,6 +23,7 @@ Users Guide
    baselines
    other_pad_algorithms
    pulse
+   mc_autoencoder_pad
    references
    resources
    api

doc/mc_autoencoder_pad.rst

+
+
+.. _bob.pad.face.mc_autoencoder_pad:
+
+
+=============================================
+ Multi-channel face PAD using autoencoders
+=============================================
+
+This section explains how to run a complete face PAD experiment using multi-channel autoencoder-based face PAD system, as well as a training work-flow.
+
+The system discussed in this section is introduced the following publication [NGM19]_. It is **strongly recommended** to check the publication for better understanding
+of the described work-flow.
+
+.. warning::
+
+   Algorithms introduced in this section might be in the process of publishing. Therefore, it is not
+   allowed to publish results introduced in this section without permission of the owner of the package.
+   If you are planning to use the results from this section, please contact the owner of the package first.
+   Please check the ``setup.py`` for contact information.
+
+
+Running face PAD Experiments
+------------------------------
+
+Please refer to :ref:`bob.pad.face.baselines` section of current documentation for more details on how to run the face PAD experiments and setup the databases.
+
+
+Training multi-channel autoencoder-based face PAD system.
+----------------------------------------------------------------
+
+As introduced in the paper [NGM19]_, the training of the system is composed of three main steps, which are summarize in the following table:
+
++----------------------+----------------------+---------------------+
+| Train step           | Training data        | DB, classes used    |
++----------------------+----------------------+---------------------+
+| Train N AEs          | RGB face regions     | CelebA, BF          |
++----------------------+----------------------+---------------------+
+| Fine-tune N AEs      | MC face regions      | WMCA, BF            |
++----------------------+----------------------+---------------------+
+| Train an MLP         | MC latent encodings  | WMCA, BF and PA     |
++----------------------+----------------------+---------------------+
+
+In the above table, **BF** and **PA** stands for samples from **bona-fide** and **presentation attack** classes.
+
+As one can conclude from the table, CelebA and WMCA databases must be installed before the training can take place.
+See :ref:`bob.pad.face.baselines` for databases installation details.
+
+
+1. Train N AEs on RGB data from CelebA
+===========================================
+
+In [NGM19]_ N autoencoders are trained, one for each facial region, here for explanatory purposes, a system containing **one** autoencoder is observed, thus N=1.
+This autoencoder is first pre-trained using RGB images of entire face, which are cropped from CelebA database.
+
+To prepare the training data one can use the following command:
+
+
+.. code-block:: sh
+
+    ./bin/spoof.py \                                        # spoof.py is used to run the preprocessor
+    celeb-a \                                               # run for CelebA database
+    lbp-svm \                                               # required by spoof.py, but unused
+    --skip-extractor-training --skip-extraction --skip-projector-training --skip-projection --skip-score-computation --allow-missing-files \    # execute only preprocessing step
+    --grid idiap \                                          # use grid, only for Idiap users, remove otherwise
+    --groups train \                                        # preprocess only training set of CelebA
+    --preprocessor rgb-face-detect-check-quality-128x128 \  # preprocessor entry point
+    --sub-directory <PATH_TO_STORE_THE_RESULTS>             # define your path here
+
+Running above command, the RGB facial images are aligned and cropped from the training set of the CelebA database. Additionally, a quality assessment is applied to each facial image.
+More specifically, an eye detection algorithm is applied to face images, assuring the deviation of eye coordinates from expected positions is not significant.
+See [NGM19]_ for more details.
+
+Once above script is completed, the data suitable for autoencoder training is located in the folder ``<PATH_TO_STORE_THE_RESULTS>/preprocessed/``. Now the autoencoder can be trained.
+The training procedure is explained in the **Convolutional autoencoder** section in the documentation of the ``bob.learn.pytorch`` package.
+
+.. note::
+
+  Functionality of ``bob.pad.face`` is used to compute the training data. Install and follow the documentation of ``bob.learn.pytorch`` to train the autoencoders. This functional decoupling helps to avoid the dependency of
+  ``bob.pad.face`` from **PyTorch**.
+
+
+.. include:: links.rst
+
+
+
+
+

doc/references.rst

@@ -18,3 +18,6 @@ References
 
 .. [CDSR17] *C. Chen, A. Dantcheva, T. Swearingen, A. Ross*, **Spoofing Faces Using Makeup: An Investigative Study**,
             in: Proc. of 3rd IEEE International Conference on Identity, Security and Behavior Analysis (ISBA), (New Delhi, India), February 2017.
+
+.. [NGM19] *O. Nikisins, A. George, S. Marcel*, **Domain Adaptation in Multi-Channel Autoencoder based Features for Robust Face Anti-Spoofing**,
+            in: Submitted to: 2019 International Conference on Biometrics (ICB), 2019.

setup.py

@@ -117,6 +117,7 @@ setup(
             'rgb-face-detect-dlib = bob.pad.face.config.preprocessor.video_face_crop:rgb_face_detector_dlib',  # detect faces locally replacing database annotations
             'rgb-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:rgb_face_detector_mtcnn',  # detect faces locally replacing database annotations
             'bw-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:bw_face_detect_mtcnn',  # detect faces locally, return BW image
+            'rgb-face-detect-check-quality-128x128 = bob.pad.face.config.preprocessor.face_feature_crop_quality_check:face_feature_0_128x128_crop_rgb',  # detect faces locally replacing database annotations, also check face quality by trying to detect the eyes in cropped face.
         ],
 
         # registered extractors: