Commit cf6daf39 authored by Olegs NIKISINS's avatar Olegs NIKISINS

Merge branch 'multi_channel_preproc' into 'master'

Multi-channel data preprocessing

See merge request !84
parents ec79e1e7 dc01bd06
Pipeline #26324 passed with stages
in 9 minutes and 46 seconds
#!/usr/bin/env python
"""
Idiap BATL DB is a database for face PAD experiments.
"""
from bob.pad.face.database import BatlPadDatabase
# Directory where the data files are stored.
# This directory is given in the .bob_bio_databases.txt file located in your home directory
ORIGINAL_DIRECTORY = "[YOUR_BATL_DB_DIRECTORY]"
"""Value of ``~/.bob_bio_databases.txt`` for this database"""
ORIGINAL_EXTENSION = ".h5" # extension of the data files
ANNOTATIONS_TEMP_DIR = "" # NOTE: this variable is NOT assigned in the instance of the BatlPadDatabase, thus "rc" functionality defined in bob.extension will be involved
PROTOCOL = 'grandtest-color*infrared*depth-10' # use 10 frames for PAD experiments
database = BatlPadDatabase(
protocol=PROTOCOL,
original_directory=ORIGINAL_DIRECTORY,
original_extension=ORIGINAL_EXTENSION,
landmark_detect_method="mtcnn", # detect annotations using mtcnn
exclude_attacks_list=['makeup'],
exclude_pai_all_sets=True, # exclude makeup from all the sets, which is the default behavior for grandtest protocol
append_color_face_roi_annot=False) # do not append annotations, defining ROI in the cropped face image, to the dictionary of annotations
"""The :py:class:`bob.pad.base.database.BatlPadDatabase` derivative with BATL Db
database settings.
.. warning::
This class only provides a programmatic interface to load data in an orderly
manner, respecting usage protocols. It does **not** contain the raw
data files. You should procure those yourself.
Notice that ``original_directory`` is set to ``[YOUR_BATL_DB_DIRECTORY]``.
You must make sure to create ``${HOME}/.bob_bio_databases.txt`` file setting this
value to the places where you actually installed the BATL Govt database.
"""
protocol = PROTOCOL
"""
You may modify this at runtime by specifying the option ``--protocol`` on the
command-line of ``spoof.py`` or using the keyword ``protocol`` on a
configuration file that is loaded **after** this configuration resource.
"""
groups = ["train", "dev", "eval"]
"""The default groups to use for reproducing the baselines.
You may modify this at runtime by specifying the option ``--groups`` on the
command-line of ``spoof.py`` or using the keyword ``groups`` on a
configuration file that is loaded **after** this configuration resource.
"""
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# =============================================================================
# Import here:
from bob.pad.face.preprocessor import VideoFaceCropAlignBlockPatch
from bob.pad.face.preprocessor import FaceCropAlign
from bob.bio.video.preprocessor import Wrapper
from bob.bio.video.utils import FrameSelector
from bob.pad.face.preprocessor.FaceCropAlign import auto_norm_image as _norm_func
from bob.pad.face.preprocessor import BlockPatch
# =============================================================================
# names of the channels to process:
_channel_names = ['color', 'infrared', 'depth']
# =============================================================================
# dictionary containing preprocessors for all channels:
_preprocessors = {}
"""
Preprocessor to be used for Color channel.
"""
FACE_SIZE = 128 # The size of the resulting face
RGB_OUTPUT_FLAG = False # BW output
USE_FACE_ALIGNMENT = True # use annotations
MAX_IMAGE_SIZE = None # no limiting here
FACE_DETECTION_METHOD = None # use ANNOTATIONS
MIN_FACE_SIZE = 50 # skip small faces
_image_preprocessor = FaceCropAlign(face_size = FACE_SIZE,
rgb_output_flag = RGB_OUTPUT_FLAG,
use_face_alignment = USE_FACE_ALIGNMENT,
max_image_size = MAX_IMAGE_SIZE,
face_detection_method = FACE_DETECTION_METHOD,
min_face_size = MIN_FACE_SIZE)
_frame_selector = FrameSelector(selection_style = "all")
_preprocessor_rgb = Wrapper(preprocessor = _image_preprocessor,
frame_selector = _frame_selector)
_preprocessors[_channel_names[0]] = _preprocessor_rgb
"""
Preprocessor to be used for Infrared (or Thermal) channels:
"""
FACE_SIZE = 128 # The size of the resulting face
RGB_OUTPUT_FLAG = False # Gray-scale output
USE_FACE_ALIGNMENT = True # use annotations
MAX_IMAGE_SIZE = None # no limiting here
FACE_DETECTION_METHOD = None # use annotations
MIN_FACE_SIZE = 50 # skip small faces
NORMALIZATION_FUNCTION = _norm_func
NORMALIZATION_FUNCTION_KWARGS = {}
NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':3.0, 'norm_method':'MAD'}
_image_preprocessor_ir = FaceCropAlign(face_size = FACE_SIZE,
rgb_output_flag = RGB_OUTPUT_FLAG,
use_face_alignment = USE_FACE_ALIGNMENT,
max_image_size = MAX_IMAGE_SIZE,
face_detection_method = FACE_DETECTION_METHOD,
min_face_size = MIN_FACE_SIZE,
normalization_function = NORMALIZATION_FUNCTION,
normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
_preprocessor_ir = Wrapper(preprocessor = _image_preprocessor_ir,
frame_selector = _frame_selector)
_preprocessors[_channel_names[1]] = _preprocessor_ir
"""
Preprocessor to be used for Depth channel:
"""
FACE_SIZE = 128 # The size of the resulting face
RGB_OUTPUT_FLAG = False # Gray-scale output
USE_FACE_ALIGNMENT = True # use annotations
MAX_IMAGE_SIZE = None # no limiting here
FACE_DETECTION_METHOD = None # use annotations
MIN_FACE_SIZE = 50 # skip small faces
NORMALIZATION_FUNCTION = _norm_func
NORMALIZATION_FUNCTION_KWARGS = {}
NORMALIZATION_FUNCTION_KWARGS = {'n_sigma':6.0, 'norm_method':'MAD'}
_image_preprocessor_d = FaceCropAlign(face_size = FACE_SIZE,
rgb_output_flag = RGB_OUTPUT_FLAG,
use_face_alignment = USE_FACE_ALIGNMENT,
max_image_size = MAX_IMAGE_SIZE,
face_detection_method = FACE_DETECTION_METHOD,
min_face_size = MIN_FACE_SIZE,
normalization_function = NORMALIZATION_FUNCTION,
normalization_function_kwargs = NORMALIZATION_FUNCTION_KWARGS)
_preprocessor_d = Wrapper(preprocessor = _image_preprocessor_d,
frame_selector = _frame_selector)
_preprocessors[_channel_names[2]] = _preprocessor_d
# =============================================================================
# define parameters and an instance of the patch extractor:
PATCH_SIZE = 128
STEP = 1
_block_patch_128x128 = BlockPatch(patch_size = PATCH_SIZE,
step = STEP,
use_annotations_flag = False)
# =============================================================================
"""
Define an instance for extraction of one (**whole face**) multi-channel
(BW-NIR-D) face patch of the size (3 x 128 x 128).
"""
video_face_crop_align_bw_ir_d_channels_3x128x128 = VideoFaceCropAlignBlockPatch(preprocessors = _preprocessors,
channel_names = _channel_names,
return_multi_channel_flag = True,
block_patch_preprocessor = _block_patch_128x128)
......@@ -240,6 +240,8 @@ class VideoFaceCropAlignBlockPatch(Preprocessor, object):
facial region. ROI is annotated as follows:
``annotations['face_roi'][0] = [x_top_left, y_top_left]``
``annotations['face_roi'][1] = [x_bottom_right, y_bottom_right]``
If ``face_roi`` annotations are undefined, the patches will be
extracted from an entire cropped facial image.
**Returns:**
......
......@@ -52,6 +52,8 @@ from bob.pad.face.config.preprocessor.face_feature_crop_quality_check import fac
from bob.pad.face.utils.patch_utils import reshape_flat_patches
from bob.pad.face.config.preprocessor.video_face_crop_align_block_patch import video_face_crop_align_bw_ir_d_channels_3x128x128 as mc_preprocessor
def test_detect_face_landmarks_in_image_mtcnn():
......@@ -335,6 +337,42 @@ def test_preproc_with_quality_check():
assert data_preprocessed is None
# =============================================================================
def test_multi_channel_preprocessing():
"""
Test video_face_crop_align_bw_ir_d_channels_3x128x128 preprocessor.
"""
# =========================================================================
# prepare the test data:
image = load(datafile('test_image.png', 'bob.pad.face.test'))
# annotations must be known for this preprocessor, so compute them:
annotations = detect_face_landmarks_in_image(image, method="mtcnn")
video_color, annotations = convert_image_to_video_data(image, annotations, 2)
video_bw, _ = convert_image_to_video_data(image[0], annotations, 2)
mc_video = {}
mc_video["color"] = video_color
mc_video["infrared"] = video_bw
mc_video["depth"] = video_bw
# =========================================================================
# test the preprocessor:
data_preprocessed = mc_preprocessor(mc_video, annotations)
assert len(data_preprocessed) == 2
assert data_preprocessed[0][1].shape == (3, 128, 128)
# chanenls are preprocessed differently, thus this should apply:
assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][1])
assert np.any(data_preprocessed[0][1][0] != data_preprocessed[0][1][2])
# =============================================================================
def test_reshape_flat_patches():
"""
......
......@@ -83,6 +83,25 @@ The training procedure is explained in the **Convolutional autoencoder** section
.. include:: links.rst
2. Fine-tune N AEs on multi-channel data from WMCA (legacy name BATL) database
=================================================================================
Following the training procedure of [NGM19]_, the autoencoders are next fine-tuned on the multi-channel (**MC**) data from WMCA.
In this example, MC training data is a stack of gray-scale, NIR, and Depth (BW-NIR-D) facial images.
To prepare the training data one can use the following command:
.. code-block:: sh
./bin/spoof.py \ # spoof.py is used to run the preprocessor
batl-db-rgb-ir-d-grandtest \ # WMCA database instance allowing to load RGB-NIR-D channels
lbp-svm \ # required by spoof.py, but unused
--skip-extractor-training --skip-extraction --skip-projector-training --skip-projection --skip-score-computation --allow-missing-files \ # execute only preprocessing step
--grid idiap \ # use grid, only for Idiap users, remove otherwise
--preprocessor video-face-crop-align-bw-ir-d-channels-3x128x128 \ # preprocessor entry point
--sub-directory <PATH_TO_STORE_THE_RESULTS> # define your path here
Once above script is completed, the MC data suitable for autoencoder fine-tuning is located in the folder ``<PATH_TO_STORE_THE_RESULTS>/preprocessed/``.
Now the autoencoder can be fine-tuned. Again, the fine-tuning procedure is explained in the **Convolutional autoencoder** section in the documentation of the ``bob.learn.pytorch`` package.
......@@ -68,10 +68,11 @@ setup(
'msu-mfsd = bob.pad.face.config.msu_mfsd:database',
'aggregated-db = bob.pad.face.config.aggregated_db:database',
'mifs = bob.pad.face.config.mifs:database',
'batl-db = bob.pad.face.config.batl_db:database',
'batl-db-infrared = bob.pad.face.config.batl_db_infrared:database',
'batl-db-depth = bob.pad.face.config.batl_db_depth:database',
'batl-db-thermal = bob.pad.face.config.batl_db_thermal:database',
'batl-db = bob.pad.face.config.database.batl.batl_db:database',
'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared:database',
'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth:database',
'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal:database',
'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest:database',
'celeb-a = bob.pad.face.config.celeb_a:database',
'maskattack = bob.pad.face.config.maskattack:database',
],
......@@ -84,10 +85,11 @@ setup(
'msu-mfsd = bob.pad.face.config.msu_mfsd',
'aggregated-db = bob.pad.face.config.aggregated_db',
'mifs = bob.pad.face.config.mifs',
'batl-db = bob.pad.face.config.batl_db',
'batl-db-infrared = bob.pad.face.config.batl_db_infrared',
'batl-db-depth = bob.pad.face.config.batl_db_depth',
'batl-db-thermal = bob.pad.face.config.batl_db_thermal',
'batl-db = bob.pad.face.config.database.batl.batl_db',
'batl-db-infrared = bob.pad.face.config.database.batl.batl_db_infrared',
'batl-db-depth = bob.pad.face.config.database.batl.batl_db_depth',
'batl-db-thermal = bob.pad.face.config.database.batl.batl_db_thermal',
'batl-db-rgb-ir-d-grandtest = bob.pad.face.config.database.batl.batl_db_rgb_ir_d_grandtest',
'celeb-a = bob.pad.face.config.celeb_a',
'maskattack = bob.pad.face.config.maskattack',
......@@ -118,6 +120,7 @@ setup(
'rgb-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:rgb_face_detector_mtcnn', # detect faces locally replacing database annotations
'bw-face-detect-mtcnn = bob.pad.face.config.preprocessor.video_face_crop:bw_face_detect_mtcnn', # detect faces locally, return BW image
'rgb-face-detect-check-quality-128x128 = bob.pad.face.config.preprocessor.face_feature_crop_quality_check:face_feature_0_128x128_crop_rgb', # detect faces locally replacing database annotations, also check face quality by trying to detect the eyes in cropped face.
'video-face-crop-align-bw-ir-d-channels-3x128x128 = bob.pad.face.config.preprocessor.video_face_crop_align_block_patch:video_face_crop_align_bw_ir_d_channels_3x128x128', # Extract a BW-NIR-D patch of size (3 x 128 x 128) containing aligned face
],
# registered extractors:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment