Merge branch 'multi_channel_preproc' into 'master'

Multi-channel data preprocessing

See merge request !84

bob/pad/face/config/database/batl/batl_db_rgb_ir_d_grandtest.py

+#!/usr/bin/env python
+"""
+Idiap BATL DB is a database for face PAD experiments.
+"""
+
+from bob.pad.face.database import BatlPadDatabase
+
+# Directory where the data files are stored.
+# This directory is given in the .bob_bio_databases.txt file located in your home directory
+ORIGINAL_DIRECTORY = "[YOUR_BATL_DB_DIRECTORY]"
+"""Value of ``~/.bob_bio_databases.txt`` for this database"""
+
+ORIGINAL_EXTENSION = ".h5"  # extension of the data files
+
+ANNOTATIONS_TEMP_DIR = ""  # NOTE: this variable is NOT assigned in the instance of the BatlPadDatabase, thus "rc" functionality defined in bob.extension will be involved
+
+PROTOCOL = 'grandtest-color*infrared*depth-10'  # use 10 frames for PAD experiments
+
+database = BatlPadDatabase(
+    protocol=PROTOCOL,
+    original_directory=ORIGINAL_DIRECTORY,
+    original_extension=ORIGINAL_EXTENSION,
+    landmark_detect_method="mtcnn",  # detect annotations using mtcnn
+    exclude_attacks_list=['makeup'],
+    exclude_pai_all_sets=True,  # exclude makeup from all the sets, which is the default behavior for grandtest protocol
+    append_color_face_roi_annot=False)  # do not append annotations, defining ROI in the cropped face image, to the dictionary of annotations
+
+"""The :py:class:`bob.pad.base.database.BatlPadDatabase` derivative with BATL Db
+database settings.
+
+.. warning::
+
+   This class only provides a programmatic interface to load data in an orderly
+   manner, respecting usage protocols. It does **not** contain the raw
+   data files. You should procure those yourself.
+
+Notice that ``original_directory`` is set to ``[YOUR_BATL_DB_DIRECTORY]``.
+You must make sure to create ``${HOME}/.bob_bio_databases.txt`` file setting this
+value to the places where you actually installed the BATL Govt database.
+"""
+
+protocol = PROTOCOL
+"""
+You may modify this at runtime by specifying the option ``--protocol`` on the
+command-line of ``spoof.py`` or using the keyword ``protocol`` on a
+configuration file that is loaded **after** this configuration resource.
+"""
+
+groups = ["train", "dev", "eval"]
+"""The default groups to use for reproducing the baselines.
+
+You may modify this at runtime by specifying the option ``--groups`` on the
+command-line of ``spoof.py`` or using the keyword ``groups`` on a
+configuration file that is loaded **after** this configuration resource.
+"""

bob/pad/face/config/preprocessor/video_face_crop_align_block_patch.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# =============================================================================
+# Import here:
+from bob.pad.face.preprocessor import VideoFaceCropAlignBlockPatch
+
+from bob.pad.face.preprocessor import FaceCropAlign
+
+from bob.bio.video.preprocessor import Wrapper
+
+from bob.bio.video.utils import FrameSelector
+
+from bob.pad.face.preprocessor.FaceCropAlign import auto_norm_image as _norm_func
+
+from bob.pad.face.preprocessor import BlockPatch
+
+
+# =============================================================================
+# names of the channels to process:
+_channel_names = ['color', 'infrared', 'depth']
+
+
+# =============================================================================
+# dictionary containing preprocessors for all channels:
+_preprocessors = {}
+
+"""
+Preprocessor to be used for Color channel.
+"""
+FACE_SIZE = 128  # The size of the resulting face
+RGB_OUTPUT_FLAG = False  # BW output
+USE_FACE_ALIGNMENT = True  # use annotations
+MAX_IMAGE_SIZE = None  # no limiting here
+FACE_DETECTION_METHOD = None  # use ANNOTATIONS
+MIN_FACE_SIZE = 50  # skip small faces
+
+_image_preprocessor = FaceCropAlign(face_size = FACE_SIZE,
+                                    rgb_output_flag = RGB_OUTPUT_FLAG,
+                                    use_face_alignment = USE_FACE_ALIGNMENT,
+                                    max_image_size = MAX_IMAGE_SIZE,
+                                    face_detection_method = FACE_DETECTION_METHOD,
+                                    min_face_size = MIN_FACE_SIZE)
+
+_frame_selector = FrameSelector(selection_style = "all")
+
+_preprocessor_rgb = Wrapper(preprocessor = _image_preprocessor,
+                            frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[0]] = _preprocessor_rgb
+
+"""
+Preprocessor to be used for Infrared (or Thermal) channels:
+"""
+FACE_SIZE = 128  # The size of the resulting face
+RGB_OUTPUT_FLAG = False  # Gray-scale output
+USE_FACE_ALIGNMENT = True  # use annotations
+MAX_IMAGE_SIZE = None  # no limiting here
+FACE_DETECTION_METHOD = None  # use annotations
+MIN_FACE_SIZE = 50  # skip small faces
+NORMALIZATION_FUNCTION = _norm_func
+NORMALIZATION_FUNCTION_KWARGS = {}
+NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':3.0, 'norm_method':'MAD'}
+
+_image_preprocessor_ir = FaceCropAlign(face_size = FACE_SIZE,
+                                    rgb_output_flag = RGB_OUTPUT_FLAG,
+                                    use_face_alignment = USE_FACE_ALIGNMENT,
+                                    max_image_size = MAX_IMAGE_SIZE,
+                                    face_detection_method = FACE_DETECTION_METHOD,
+                                    min_face_size = MIN_FACE_SIZE,
+                                    normalization_function = NORMALIZATION_FUNCTION,
+                                    normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
+
+_preprocessor_ir = Wrapper(preprocessor = _image_preprocessor_ir,
+                               frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[1]] = _preprocessor_ir
+
+"""
+Preprocessor to be used for Depth channel:
+"""
+FACE_SIZE = 128  # The size of the resulting face
+RGB_OUTPUT_FLAG = False  # Gray-scale output
+USE_FACE_ALIGNMENT = True  # use annotations
+MAX_IMAGE_SIZE = None  # no limiting here
+FACE_DETECTION_METHOD = None  # use annotations
+MIN_FACE_SIZE = 50  # skip small faces
+NORMALIZATION_FUNCTION = _norm_func
+NORMALIZATION_FUNCTION_KWARGS = {}
+NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':6.0, 'norm_method':'MAD'}
+
+_image_preprocessor_d = FaceCropAlign(face_size = FACE_SIZE,
+                                    rgb_output_flag = RGB_OUTPUT_FLAG,
+                                    use_face_alignment = USE_FACE_ALIGNMENT,
+                                    max_image_size = MAX_IMAGE_SIZE,
+                                    face_detection_method = FACE_DETECTION_METHOD,
+                                    min_face_size = MIN_FACE_SIZE,
+                                    normalization_function = NORMALIZATION_FUNCTION,
+                                    normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
+
+_preprocessor_d = Wrapper(preprocessor = _image_preprocessor_d,
+                               frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[2]] = _preprocessor_d
+
+
+# =============================================================================
+# define parameters and an instance of the patch extractor:
+PATCH_SIZE = 128
+STEP = 1
+
+_block_patch_128x128 = BlockPatch(patch_size = PATCH_SIZE,
+                                  step = STEP,
+                                  use_annotations_flag = False)
+
+
+# =============================================================================
+"""
+Define an instance for extraction of one (**whole face**) multi-channel
+(BW-NIR-D) face patch of the size (3 x 128 x 128).
+"""
+video_face_crop_align_bw_ir_d_channels_3x128x128 = VideoFaceCropAlignBlockPatch(preprocessors = _preprocessors,
+                                                                                channel_names = _channel_names,
+                                                                                return_multi_channel_flag = True,
+                                                                                block_patch_preprocessor = _block_patch_128x128)
+

bob/pad/face/preprocessor/VideoFaceCropAlignBlockPatch.py

@@ -240,6 +240,8 @@ class VideoFaceCropAlignBlockPatch(Preprocessor, object):
             facial region. ROI is annotated as follows:
             ``annotations['face_roi'][0] = [x_top_left, y_top_left]``
             ``annotations['face_roi'][1] = [x_bottom_right, y_bottom_right]``
+            If ``face_roi`` annotations are undefined, the patches will be
+            extracted from an entire cropped facial image.
 
         **Returns:**
 

bob/pad/face/test/test.py

@@ -52,6 +52,8 @@ from bob.pad.face.config.preprocessor.face_feature_crop_quality_check import fac
 
 from bob.pad.face.utils.patch_utils import reshape_flat_patches
 
+from bob.pad.face.config.preprocessor.video_face_crop_align_block_patch import video_face_crop_align_bw_ir_d_channels_3x128x128 as mc_preprocessor
+
 
 def test_detect_face_landmarks_in_image_mtcnn():
 
@@ -335,6 +337,42 @@ def test_preproc_with_quality_check():
     assert data_preprocessed is None
 
 
+# =============================================================================
+def test_multi_channel_preprocessing():
+    """
+    Test video_face_crop_align_bw_ir_d_channels_3x128x128 preprocessor.
+    """
+
+    # =========================================================================
+    # prepare the test data:
+
+    image = load(datafile('test_image.png', 'bob.pad.face.test'))
+
+    # annotations must be known for this preprocessor, so compute them:
+    annotations = detect_face_landmarks_in_image(image, method="mtcnn")
+
+    video_color, annotations = convert_image_to_video_data(image, annotations, 2)
+
+    video_bw, _ = convert_image_to_video_data(image[0], annotations, 2)
+
+    mc_video = {}
+    mc_video["color"] = video_color
+    mc_video["infrared"] = video_bw
+    mc_video["depth"] = video_bw
+
+    # =========================================================================
+    # test the preprocessor:
+
+    data_preprocessed = mc_preprocessor(mc_video, annotations)
+
+    assert len(data_preprocessed) == 2
+    assert data_preprocessed[0][1].shape == (3, 128, 128)
+
+    # chanenls are preprocessed differently, thus this should apply:
+    assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][1])
+    assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][2])
+
+
 # =============================================================================
 def test_reshape_flat_patches():
     """

doc/mc_autoencoder_pad.rst

@@ -83,6 +83,25 @@ The training procedure is explained in the **Convolutional autoencoder** section
 .. include:: links.rst
 
 
+2. Fine-tune N AEs on multi-channel data from WMCA (legacy name BATL) database
+=================================================================================
 
+Following the training procedure of [NGM19]_, the autoencoders are next fine-tuned on the multi-channel (**MC**) data from WMCA.
+In this example, MC training data is a stack of gray-scale, NIR, and Depth (BW-NIR-D) facial images.
 
+To prepare the training data one can use the following command:
+
+
+.. code-block:: sh
+
+    ./bin/spoof.py \                                                    # spoof.py is used to run the preprocessor
+    batl-db-rgb-ir-d-grandtest \                                        # WMCA database instance allowing to load RGB-NIR-D channels
+    lbp-svm \                                                           # required by spoof.py, but unused
+    --skip-extractor-training --skip-extraction --skip-projector-training --skip-projection --skip-score-computation --allow-missing-files \    # execute only preprocessing step
+    --grid idiap \                                                      # use grid, only for Idiap users, remove otherwise
+    --preprocessor video-face-crop-align-bw-ir-d-channels-3x128x128 \   # preprocessor entry point
+    --sub-directory <PATH_TO_STORE_THE_RESULTS>                         # define your path here
+
+Once above script is completed, the MC data suitable for autoencoder fine-tuning is located in the folder ``<PATH_TO_STORE_THE_RESULTS>/preprocessed/``.
+Now the autoencoder can be fine-tuned. Again, the fine-tuning procedure is explained in the **Convolutional autoencoder** section in the documentation of the ``bob.learn.pytorch`` package.
 

setup.py

@@ -68,10 +68,11 @@ setup(
             'msu-mfsd = bob.pad.face.config.msu_mfsd:database',
             'aggregated-db = bob.pad.face.config.aggregated_db:database',
             'mifs = bob.pad.face.config.mifs:database',
-            'batl-db = bob.pad.face.config.batl_db:database',
-            'batl-db-infrared = bob.pad.face.config.batl_db_infrared:database',
-            'batl-db-depth = bob.pad.face.config.batl_db_depth:database',
-            'batl-db-thermal = bob.pad.face.config.batl_db_thermal:database',
+            'batl-db = bob.pad.face.config.database.batl.batl_db:database',
+            'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared:database',
+            'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth:database',
+            'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal:database',
+            'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest:database',
             'celeb-a = bob.pad.face.config.celeb_a:database',
             'maskattack = bob.pad.face.config.maskattack:database',
         ],
@@ -84,10 +85,11 @@ setup(
             'msu-mfsd = bob.pad.face.config.msu_mfsd',
             'aggregated-db = bob.pad.face.config.aggregated_db',
             'mifs = bob.pad.face.config.mifs',
-            'batl-db = bob.pad.face.config.batl_db',
-            'batl-db-infrared = bob.pad.face.config.batl_db_infrared',
-            'batl-db-depth = bob.pad.face.config.batl_db_depth',
-            'batl-db-thermal = bob.pad.face.config.batl_db_thermal',
+            'batl-db = bob.pad.face.config.database.batl.batl_db',
+            'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared',
+            'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth',
+            'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal',
+            'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest',
             'celeb-a = bob.pad.face.config.celeb_a',
             'maskattack = bob.pad.face.config.maskattack',
 
@@ -118,6 +120,7 @@ setup(
             'rgb-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:rgb_face_detector_mtcnn',  # detect faces locally replacing database annotations
             'bw-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:bw_face_detect_mtcnn',  # detect faces locally, return BW image
             'rgb-face-detect-check-quality-128x128 = bob.pad.face.config.preprocessor.face_feature_crop_quality_check:face_feature_0_128x128_crop_rgb',  # detect faces locally replacing database annotations, also check face quality by trying to detect the eyes in cropped face.
+            'video-face-crop-align-bw-ir-d-channels-3x128x128 = bob.pad.face.config.preprocessor.video_face_crop_align_block_patch:video_face_crop_align_bw_ir_d_channels_3x128x128', # Extract a BW-NIR-D patch of size (3 x 128 x 128) containing aligned face
         ],
 
         # registered extractors: