diff --git a/bob/pad/face/config/database/__init__.py b/bob/pad/face/config/database/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/bob/pad/face/config/database/batl/__init__.py b/bob/pad/face/config/database/batl/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/bob/pad/face/config/batl_db.py b/bob/pad/face/config/database/batl/batl_db.py
similarity index 100%
rename from bob/pad/face/config/batl_db.py
rename to bob/pad/face/config/database/batl/batl_db.py
diff --git a/bob/pad/face/config/batl_db_depth.py b/bob/pad/face/config/database/batl/batl_db_depth.py
similarity index 100%
rename from bob/pad/face/config/batl_db_depth.py
rename to bob/pad/face/config/database/batl/batl_db_depth.py
diff --git a/bob/pad/face/config/batl_db_infrared.py b/bob/pad/face/config/database/batl/batl_db_infrared.py
similarity index 100%
rename from bob/pad/face/config/batl_db_infrared.py
rename to bob/pad/face/config/database/batl/batl_db_infrared.py
diff --git a/bob/pad/face/config/database/batl/batl_db_rgb_ir_d_grandtest.py b/bob/pad/face/config/database/batl/batl_db_rgb_ir_d_grandtest.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fc2381bca64f7327b7f632458c574061af054f0
--- /dev/null
+++ b/bob/pad/face/config/database/batl/batl_db_rgb_ir_d_grandtest.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Idiap BATL DB is a database for face PAD experiments.
+"""
+
+from bob.pad.face.database import BatlPadDatabase
+
+# Directory where the data files are stored.
+# This directory is given in the .bob_bio_databases.txt file located in your home directory
+ORIGINAL_DIRECTORY = "[YOUR_BATL_DB_DIRECTORY]"
+"""Value of ``~/.bob_bio_databases.txt`` for this database"""
+
+ORIGINAL_EXTENSION = ".h5" # extension of the data files
+
+ANNOTATIONS_TEMP_DIR = "" # NOTE: this variable is NOT assigned in the instance of the BatlPadDatabase, thus "rc" functionality defined in bob.extension will be involved
+
+PROTOCOL = 'grandtest-color*infrared*depth-10' # use 10 frames for PAD experiments
+
+database = BatlPadDatabase(
+ protocol=PROTOCOL,
+ original_directory=ORIGINAL_DIRECTORY,
+ original_extension=ORIGINAL_EXTENSION,
+ landmark_detect_method="mtcnn", # detect annotations using mtcnn
+ exclude_attacks_list=['makeup'],
+ exclude_pai_all_sets=True, # exclude makeup from all the sets, which is the default behavior for grandtest protocol
+ append_color_face_roi_annot=False) # do not append annotations, defining ROI in the cropped face image, to the dictionary of annotations
+
+"""The :py:class:`bob.pad.base.database.BatlPadDatabase` derivative with BATL Db
+database settings.
+
+.. warning::
+
+ This class only provides a programmatic interface to load data in an orderly
+ manner, respecting usage protocols. It does **not** contain the raw
+ data files. You should procure those yourself.
+
+Notice that ``original_directory`` is set to ``[YOUR_BATL_DB_DIRECTORY]``.
+You must make sure to create ``${HOME}/.bob_bio_databases.txt`` file setting this
+value to the places where you actually installed the BATL Govt database.
+"""
+
+protocol = PROTOCOL
+"""
+You may modify this at runtime by specifying the option ``--protocol`` on the
+command-line of ``spoof.py`` or using the keyword ``protocol`` on a
+configuration file that is loaded **after** this configuration resource.
+"""
+
+groups = ["train", "dev", "eval"]
+"""The default groups to use for reproducing the baselines.
+
+You may modify this at runtime by specifying the option ``--groups`` on the
+command-line of ``spoof.py`` or using the keyword ``groups`` on a
+configuration file that is loaded **after** this configuration resource.
+"""
diff --git a/bob/pad/face/config/batl_db_thermal.py b/bob/pad/face/config/database/batl/batl_db_thermal.py
similarity index 100%
rename from bob/pad/face/config/batl_db_thermal.py
rename to bob/pad/face/config/database/batl/batl_db_thermal.py
diff --git a/bob/pad/face/config/preprocessor/video_face_crop_align_block_patch.py b/bob/pad/face/config/preprocessor/video_face_crop_align_block_patch.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa96bdf5b04834e4e550f122166c10050c1b7a3b
--- /dev/null
+++ b/bob/pad/face/config/preprocessor/video_face_crop_align_block_patch.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# =============================================================================
+# Import here:
+from bob.pad.face.preprocessor import VideoFaceCropAlignBlockPatch
+
+from bob.pad.face.preprocessor import FaceCropAlign
+
+from bob.bio.video.preprocessor import Wrapper
+
+from bob.bio.video.utils import FrameSelector
+
+from bob.pad.face.preprocessor.FaceCropAlign import auto_norm_image as _norm_func
+
+from bob.pad.face.preprocessor import BlockPatch
+
+
+# =============================================================================
+# names of the channels to process:
+_channel_names = ['color', 'infrared', 'depth']
+
+
+# =============================================================================
+# dictionary containing preprocessors for all channels:
+_preprocessors = {}
+
+"""
+Preprocessor to be used for Color channel.
+"""
+FACE_SIZE = 128 # The size of the resulting face
+RGB_OUTPUT_FLAG = False # BW output
+USE_FACE_ALIGNMENT = True # use annotations
+MAX_IMAGE_SIZE = None # no limiting here
+FACE_DETECTION_METHOD = None # use ANNOTATIONS
+MIN_FACE_SIZE = 50 # skip small faces
+
+_image_preprocessor = FaceCropAlign(face_size = FACE_SIZE,
+ rgb_output_flag = RGB_OUTPUT_FLAG,
+ use_face_alignment = USE_FACE_ALIGNMENT,
+ max_image_size = MAX_IMAGE_SIZE,
+ face_detection_method = FACE_DETECTION_METHOD,
+ min_face_size = MIN_FACE_SIZE)
+
+_frame_selector = FrameSelector(selection_style = "all")
+
+_preprocessor_rgb = Wrapper(preprocessor = _image_preprocessor,
+ frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[0]] = _preprocessor_rgb
+
+"""
+Preprocessor to be used for Infrared (or Thermal) channels:
+"""
+FACE_SIZE = 128 # The size of the resulting face
+RGB_OUTPUT_FLAG = False # Gray-scale output
+USE_FACE_ALIGNMENT = True # use annotations
+MAX_IMAGE_SIZE = None # no limiting here
+FACE_DETECTION_METHOD = None # use annotations
+MIN_FACE_SIZE = 50 # skip small faces
+NORMALIZATION_FUNCTION = _norm_func
+NORMALIZATION_FUNCTION_KWARGS = {}
+NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':3.0, 'norm_method':'MAD'}
+
+_image_preprocessor_ir = FaceCropAlign(face_size = FACE_SIZE,
+ rgb_output_flag = RGB_OUTPUT_FLAG,
+ use_face_alignment = USE_FACE_ALIGNMENT,
+ max_image_size = MAX_IMAGE_SIZE,
+ face_detection_method = FACE_DETECTION_METHOD,
+ min_face_size = MIN_FACE_SIZE,
+ normalization_function = NORMALIZATION_FUNCTION,
+ normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
+
+_preprocessor_ir = Wrapper(preprocessor = _image_preprocessor_ir,
+ frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[1]] = _preprocessor_ir
+
+"""
+Preprocessor to be used for Depth channel:
+"""
+FACE_SIZE = 128 # The size of the resulting face
+RGB_OUTPUT_FLAG = False # Gray-scale output
+USE_FACE_ALIGNMENT = True # use annotations
+MAX_IMAGE_SIZE = None # no limiting here
+FACE_DETECTION_METHOD = None # use annotations
+MIN_FACE_SIZE = 50 # skip small faces
+NORMALIZATION_FUNCTION = _norm_func
+NORMALIZATION_FUNCTION_KWARGS = {}
+NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':6.0, 'norm_method':'MAD'}
+
+_image_preprocessor_d = FaceCropAlign(face_size = FACE_SIZE,
+ rgb_output_flag = RGB_OUTPUT_FLAG,
+ use_face_alignment = USE_FACE_ALIGNMENT,
+ max_image_size = MAX_IMAGE_SIZE,
+ face_detection_method = FACE_DETECTION_METHOD,
+ min_face_size = MIN_FACE_SIZE,
+ normalization_function = NORMALIZATION_FUNCTION,
+ normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
+
+_preprocessor_d = Wrapper(preprocessor = _image_preprocessor_d,
+ frame_selector = _frame_selector)
+
+_preprocessors[_channel_names[2]] = _preprocessor_d
+
+
+# =============================================================================
+# define parameters and an instance of the patch extractor:
+PATCH_SIZE = 128
+STEP = 1
+
+_block_patch_128x128 = BlockPatch(patch_size = PATCH_SIZE,
+ step = STEP,
+ use_annotations_flag = False)
+
+
+# =============================================================================
+"""
+Define an instance for extraction of one (**whole face**) multi-channel
+(BW-NIR-D) face patch of the size (3 x 128 x 128).
+"""
+video_face_crop_align_bw_ir_d_channels_3x128x128 = VideoFaceCropAlignBlockPatch(preprocessors = _preprocessors,
+ channel_names = _channel_names,
+ return_multi_channel_flag = True,
+ block_patch_preprocessor = _block_patch_128x128)
+
diff --git a/bob/pad/face/preprocessor/VideoFaceCropAlignBlockPatch.py b/bob/pad/face/preprocessor/VideoFaceCropAlignBlockPatch.py
index 4aa913090e9272739d879a184f077637511a0884..9ca1e5cfa1fc124e55d59f78cb24b0d1bc0ce83c 100644
--- a/bob/pad/face/preprocessor/VideoFaceCropAlignBlockPatch.py
+++ b/bob/pad/face/preprocessor/VideoFaceCropAlignBlockPatch.py
@@ -240,6 +240,8 @@ class VideoFaceCropAlignBlockPatch(Preprocessor, object):
facial region. ROI is annotated as follows:
``annotations['face_roi'][0] = [x_top_left, y_top_left]``
``annotations['face_roi'][1] = [x_bottom_right, y_bottom_right]``
+ If ``face_roi`` annotations are undefined, the patches will be
+ extracted from an entire cropped facial image.
**Returns:**
diff --git a/bob/pad/face/test/test.py b/bob/pad/face/test/test.py
index cf10dcab76da3c08d8f8e27954b8a60babf8bc77..ed6c8d2c9fca05e9e24424daa2e55d2ebae31d9d 100644
--- a/bob/pad/face/test/test.py
+++ b/bob/pad/face/test/test.py
@@ -52,6 +52,8 @@ from bob.pad.face.config.preprocessor.face_feature_crop_quality_check import fac
from bob.pad.face.utils.patch_utils import reshape_flat_patches
+from bob.pad.face.config.preprocessor.video_face_crop_align_block_patch import video_face_crop_align_bw_ir_d_channels_3x128x128 as mc_preprocessor
+
def test_detect_face_landmarks_in_image_mtcnn():
@@ -335,6 +337,42 @@ def test_preproc_with_quality_check():
assert data_preprocessed is None
+# =============================================================================
+def test_multi_channel_preprocessing():
+ """
+ Test video_face_crop_align_bw_ir_d_channels_3x128x128 preprocessor.
+ """
+
+ # =========================================================================
+ # prepare the test data:
+
+ image = load(datafile('test_image.png', 'bob.pad.face.test'))
+
+ # annotations must be known for this preprocessor, so compute them:
+ annotations = detect_face_landmarks_in_image(image, method="mtcnn")
+
+ video_color, annotations = convert_image_to_video_data(image, annotations, 2)
+
+ video_bw, _ = convert_image_to_video_data(image[0], annotations, 2)
+
+ mc_video = {}
+ mc_video["color"] = video_color
+ mc_video["infrared"] = video_bw
+ mc_video["depth"] = video_bw
+
+ # =========================================================================
+ # test the preprocessor:
+
+ data_preprocessed = mc_preprocessor(mc_video, annotations)
+
+ assert len(data_preprocessed) == 2
+ assert data_preprocessed[0][1].shape == (3, 128, 128)
+
+ # chanenls are preprocessed differently, thus this should apply:
+ assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][1])
+ assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][2])
+
+
# =============================================================================
def test_reshape_flat_patches():
"""
diff --git a/doc/mc_autoencoder_pad.rst b/doc/mc_autoencoder_pad.rst
index f59cbdf19bc1726e218e6b5eb81fdeb34f9c300c..7cd6d3bee6947d2d43dd2f4a09f61bce7af5b7fb 100644
--- a/doc/mc_autoencoder_pad.rst
+++ b/doc/mc_autoencoder_pad.rst
@@ -83,6 +83,25 @@ The training procedure is explained in the **Convolutional autoencoder** section
.. include:: links.rst
+2. Fine-tune N AEs on multi-channel data from WMCA (legacy name BATL) database
+=================================================================================
+Following the training procedure of [NGM19]_, the autoencoders are next fine-tuned on the multi-channel (**MC**) data from WMCA.
+In this example, MC training data is a stack of gray-scale, NIR, and Depth (BW-NIR-D) facial images.
+To prepare the training data one can use the following command:
+
+
+.. code-block:: sh
+
+ ./bin/spoof.py \ # spoof.py is used to run the preprocessor
+ batl-db-rgb-ir-d-grandtest \ # WMCA database instance allowing to load RGB-NIR-D channels
+ lbp-svm \ # required by spoof.py, but unused
+ --skip-extractor-training --skip-extraction --skip-projector-training --skip-projection --skip-score-computation --allow-missing-files \ # execute only preprocessing step
+ --grid idiap \ # use grid, only for Idiap users, remove otherwise
+ --preprocessor video-face-crop-align-bw-ir-d-channels-3x128x128 \ # preprocessor entry point
+ --sub-directory <PATH_TO_STORE_THE_RESULTS> # define your path here
+
+Once above script is completed, the MC data suitable for autoencoder fine-tuning is located in the folder ``<PATH_TO_STORE_THE_RESULTS>/preprocessed/``.
+Now the autoencoder can be fine-tuned. Again, the fine-tuning procedure is explained in the **Convolutional autoencoder** section in the documentation of the ``bob.learn.pytorch`` package.
diff --git a/setup.py b/setup.py
index b9b97d29d3ad99ddfac96f79354377505194ae15..bdd290b8954e7a34de517cdd0ff11462021d1ff4 100644
--- a/setup.py
+++ b/setup.py
@@ -68,10 +68,11 @@ setup(
'msu-mfsd = bob.pad.face.config.msu_mfsd:database',
'aggregated-db = bob.pad.face.config.aggregated_db:database',
'mifs = bob.pad.face.config.mifs:database',
- 'batl-db = bob.pad.face.config.batl_db:database',
- 'batl-db-infrared = bob.pad.face.config.batl_db_infrared:database',
- 'batl-db-depth = bob.pad.face.config.batl_db_depth:database',
- 'batl-db-thermal = bob.pad.face.config.batl_db_thermal:database',
+ 'batl-db = bob.pad.face.config.database.batl.batl_db:database',
+ 'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared:database',
+ 'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth:database',
+ 'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal:database',
+ 'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest:database',
'celeb-a = bob.pad.face.config.celeb_a:database',
'maskattack = bob.pad.face.config.maskattack:database',
],
@@ -84,10 +85,11 @@ setup(
'msu-mfsd = bob.pad.face.config.msu_mfsd',
'aggregated-db = bob.pad.face.config.aggregated_db',
'mifs = bob.pad.face.config.mifs',
- 'batl-db = bob.pad.face.config.batl_db',
- 'batl-db-infrared = bob.pad.face.config.batl_db_infrared',
- 'batl-db-depth = bob.pad.face.config.batl_db_depth',
- 'batl-db-thermal = bob.pad.face.config.batl_db_thermal',
+ 'batl-db = bob.pad.face.config.database.batl.batl_db',
+ 'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared',
+ 'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth',
+ 'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal',
+ 'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest',
'celeb-a = bob.pad.face.config.celeb_a',
'maskattack = bob.pad.face.config.maskattack',
@@ -118,6 +120,7 @@ setup(
'rgb-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:rgb_face_detector_mtcnn', # detect faces locally replacing database annotations
'bw-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:bw_face_detect_mtcnn', # detect faces locally, return BW image
'rgb-face-detect-check-quality-128x128 = bob.pad.face.config.preprocessor.face_feature_crop_quality_check:face_feature_0_128x128_crop_rgb', # detect faces locally replacing database annotations, also check face quality by trying to detect the eyes in cropped face.
+ 'video-face-crop-align-bw-ir-d-channels-3x128x128 = bob.pad.face.config.preprocessor.video_face_crop_align_block_patch:video_face_crop_align_bw_ir_d_channels_3x128x128', # Extract a BW-NIR-D patch of size (3 x 128 x 128) containing aligned face
],
# registered extractors: