From 8866cdc3ce29cac18bcafdc427382ecea7cb0e2f Mon Sep 17 00:00:00 2001
From: Olegs NIKISINS <onikisins@italix03.idiap.ch>
Date: Wed, 31 May 2017 10:28:27 +0200
Subject: [PATCH] Added preprocessor identical to Ivana's paper
---
.../face/algorithm/VideoSvmPadAlgorithm.py | 14 ++-
.../algorithm/video_svm_pad_algorithm.py | 43 +++++--
.../config/preprocessor/video_face_crop.py | 46 ++++++-
bob/pad/face/preprocessor/ImageFaceCrop.py | 116 ++++++++++++++++++
bob/pad/face/preprocessor/VideoFaceCrop.py | 107 +++++++++++++---
bob/pad/face/preprocessor/__init__.py | 2 +
setup.py | 4 +
7 files changed, 295 insertions(+), 37 deletions(-)
create mode 100644 bob/pad/face/preprocessor/ImageFaceCrop.py
diff --git a/bob/pad/face/algorithm/VideoSvmPadAlgorithm.py b/bob/pad/face/algorithm/VideoSvmPadAlgorithm.py
index ef921338..e99f7572 100644
--- a/bob/pad/face/algorithm/VideoSvmPadAlgorithm.py
+++ b/bob/pad/face/algorithm/VideoSvmPadAlgorithm.py
@@ -57,7 +57,7 @@ class VideoSvmPadAlgorithm(Algorithm):
``mean_std_norm_flag`` : :py:class:`bool`
Perform mean-std normalization of data if set to True. Default: False.
- ``frame_level_scores`` : :py:class:`bool`
+ ``frame_level_scores_flag`` : :py:class:`bool`
Return scores for each frame individually if True. Otherwise, return a
single score per video. Default: False.
"""
@@ -68,7 +68,7 @@ class VideoSvmPadAlgorithm(Algorithm):
n_samples = 10000,
trainer_grid_search_params = { 'cost': [2**p for p in range(-5, 16, 2)], 'gamma': [2**p for p in range(-15, 4, 2)]},
mean_std_norm_flag = False,
- frame_level_scores = False):
+ frame_level_scores_flag = False):
Algorithm.__init__(self,
@@ -77,7 +77,7 @@ class VideoSvmPadAlgorithm(Algorithm):
n_samples = n_samples,
trainer_grid_search_params = trainer_grid_search_params,
mean_std_norm_flag = mean_std_norm_flag,
- frame_level_scores = frame_level_scores,
+ frame_level_scores_flag = frame_level_scores_flag,
performs_projection=True,
requires_projector_training=True)
@@ -86,7 +86,7 @@ class VideoSvmPadAlgorithm(Algorithm):
self.n_samples = n_samples
self.trainer_grid_search_params = trainer_grid_search_params
self.mean_std_norm_flag = mean_std_norm_flag
- self.frame_level_scores = frame_level_scores
+ self.frame_level_scores_flag = frame_level_scores_flag
self.machine = None
@@ -668,6 +668,8 @@ class VideoSvmPadAlgorithm(Algorithm):
probabilities = self.machine.predict_class_and_probabilities(features_array)[1]
+# probabilities = self.machine.predict_class_and_scores(features_array)[1]
+
return probabilities
@@ -691,7 +693,7 @@ class VideoSvmPadAlgorithm(Algorithm):
A probability of a sample being a real class.
"""
- if self.frame_level_scores:
+ if self.frame_level_scores_flag:
score = toscore[:,0] # here score is a list containing scores for each frame
@@ -720,7 +722,7 @@ class VideoSvmPadAlgorithm(Algorithm):
# import ipdb; ipdb.set_trace()
- if self.frame_level_scores:
+ if self.frame_level_scores_flag:
list_of_scores = self.score(toscore)
diff --git a/bob/pad/face/config/algorithm/video_svm_pad_algorithm.py b/bob/pad/face/config/algorithm/video_svm_pad_algorithm.py
index 6fe78da7..cd6fc1ad 100644
--- a/bob/pad/face/config/algorithm/video_svm_pad_algorithm.py
+++ b/bob/pad/face/config/algorithm/video_svm_pad_algorithm.py
@@ -9,20 +9,41 @@ from bob.pad.face.algorithm import VideoSvmPadAlgorithm
machine_type = 'C_SVC'
kernel_type = 'RBF'
n_samples = 10000
-trainer_grid_search_params = { 'cost': [2**p for p in range(-5, 16, 2)], 'gamma': [2**p for p in range(-15, 4, 2)]}
-# trainer_grid_search_params = { 'cost': [32768], 'gamma': [2] }
+# trainer_grid_search_params = {'cost': [2**p for p in range(-5, 16, 2)], 'gamma': [2**p for p in range(-15, 4, 2)]}
+trainer_grid_search_params = {'cost': [2**p for p in range(-3, 14, 2)], 'gamma': [2**p for p in range(-15, 0, 2)]}
mean_std_norm_flag = False
+frame_level_scores_flag = False # one score per video(!) in this case
video_svm_pad_algorithm_10k_grid = VideoSvmPadAlgorithm(machine_type = machine_type,
- kernel_type = kernel_type,
- n_samples = n_samples,
- trainer_grid_search_params = trainer_grid_search_params,
- mean_std_norm_flag = mean_std_norm_flag)
+ kernel_type = kernel_type,
+ n_samples = n_samples,
+ trainer_grid_search_params = trainer_grid_search_params,
+ mean_std_norm_flag = mean_std_norm_flag,
+ frame_level_scores_flag = frame_level_scores_flag)
-mean_std_norm_flag = True
+mean_std_norm_flag = True # enable mean-std normalization
video_svm_pad_algorithm_10k_grid_mean_std = VideoSvmPadAlgorithm(machine_type = machine_type,
- kernel_type = kernel_type,
- n_samples = n_samples,
- trainer_grid_search_params = trainer_grid_search_params,
- mean_std_norm_flag = mean_std_norm_flag)
\ No newline at end of file
+ kernel_type = kernel_type,
+ n_samples = n_samples,
+ trainer_grid_search_params = trainer_grid_search_params,
+ mean_std_norm_flag = mean_std_norm_flag,
+ frame_level_scores_flag = frame_level_scores_flag)
+
+frame_level_scores_flag = True # one score per frame(!) in this case
+
+video_svm_pad_algorithm_10k_grid_mean_std_frame_level = VideoSvmPadAlgorithm(machine_type = machine_type,
+ kernel_type = kernel_type,
+ n_samples = n_samples,
+ trainer_grid_search_params = trainer_grid_search_params,
+ mean_std_norm_flag = mean_std_norm_flag,
+ frame_level_scores_flag = frame_level_scores_flag)
+
+trainer_grid_search_params = {'cost': [1], 'gamma': [0]} # set the default LibSVM parameters
+
+video_svm_pad_algorithm_default_svm_param_mean_std_frame_level = VideoSvmPadAlgorithm(machine_type = machine_type,
+ kernel_type = kernel_type,
+ n_samples = n_samples,
+ trainer_grid_search_params = trainer_grid_search_params,
+ mean_std_norm_flag = mean_std_norm_flag,
+ frame_level_scores_flag = frame_level_scores_flag)
\ No newline at end of file
diff --git a/bob/pad/face/config/preprocessor/video_face_crop.py b/bob/pad/face/config/preprocessor/video_face_crop.py
index 61eaf725..c4d00efc 100644
--- a/bob/pad/face/config/preprocessor/video_face_crop.py
+++ b/bob/pad/face/config/preprocessor/video_face_crop.py
@@ -12,12 +12,46 @@ fixed_positions = None
mask_sigma = None # The sigma for random values areas outside image
mask_neighbors = 5 # The number of neighbors to consider while extrapolating
mask_seed = None # The seed for generating random values during extrapolation
+check_face_size_flag = False # Don't check the size of the face
+min_face_size = 50
+use_local_cropper_flag = False # Use the cropper of bob.bio.face
color_channel = 'gray' # Convert image to gray-scale format
video_face_crop_preproc_64_64 = VideoFaceCrop(cropped_image_size = cropped_image_size,
- cropped_positions = cropped_positions,
- fixed_positions = fixed_positions,
- mask_sigma = mask_sigma,
- mask_neighbors = mask_neighbors,
- mask_seed = None,
- color_channel = color_channel)
+ cropped_positions = cropped_positions,
+ fixed_positions = fixed_positions,
+ mask_sigma = mask_sigma,
+ mask_neighbors = mask_neighbors,
+ mask_seed = None,
+ check_face_size_flag = check_face_size_flag,
+ min_face_size = min_face_size,
+ use_local_cropper_flag = use_local_cropper_flag,
+ color_channel = color_channel)
+
+check_face_size_flag = True # Check the size of the face
+min_face_size = 50
+
+video_face_crop_preproc_64_64_face_50 = VideoFaceCrop(cropped_image_size = cropped_image_size,
+ cropped_positions = cropped_positions,
+ fixed_positions = fixed_positions,
+ mask_sigma = mask_sigma,
+ mask_neighbors = mask_neighbors,
+ mask_seed = None,
+ check_face_size_flag = check_face_size_flag,
+ min_face_size = min_face_size,
+ use_local_cropper_flag = use_local_cropper_flag,
+ color_channel = color_channel)
+
+
+use_local_cropper_flag = True # Use the local face cropping class (identical to Ivana's paper)
+
+video_face_crop_preproc_64_64_face_50_local_cropper = VideoFaceCrop(cropped_image_size = cropped_image_size,
+ cropped_positions = cropped_positions,
+ fixed_positions = fixed_positions,
+ mask_sigma = mask_sigma,
+ mask_neighbors = mask_neighbors,
+ mask_seed = None,
+ check_face_size_flag = check_face_size_flag,
+ min_face_size = min_face_size,
+ use_local_cropper_flag = use_local_cropper_flag,
+ color_channel = color_channel)
\ No newline at end of file
diff --git a/bob/pad/face/preprocessor/ImageFaceCrop.py b/bob/pad/face/preprocessor/ImageFaceCrop.py
new file mode 100644
index 00000000..a59c362a
--- /dev/null
+++ b/bob/pad/face/preprocessor/ImageFaceCrop.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+Created on Tue May 30 14:11:16 2017
+
+@author: Olegs Nikisins
+"""
+
+#==============================================================================
+# Import what is needed here:
+
+from bob.bio.base.preprocessor import Preprocessor
+
+import numpy as np
+
+import bob.ip.color
+
+import bob.ip.base
+
+#==============================================================================
+# Main body:
+
+class ImageFaceCrop(Preprocessor):
+ """
+ This class crops the face in the input image given annotations defining
+ the face bounding box. The size of the face is also normalized to the
+ pre-defined dimensions. If input image is RGB it is first converted to the
+ gray-scale format.
+ The algorithm is identical to the following paper:
+ "On the Effectiveness of Local Binary Patterns in Face Anti-spoofing"
+
+ **Parameters:**
+
+ ``face_size`` : :py:class:`int`
+ The size of the face after normalization.
+ """
+ #==========================================================================
+ def __init__(self, face_size):
+
+
+ Preprocessor.__init__(self,
+ face_size = face_size)
+
+ self.face_size = face_size
+
+
+ #==========================================================================
+ def normalize_image_size(self, image, annotations, face_size):
+ """
+ This function crops the face in the input image given annotations defining
+ the face bounding box. The size of the face is also normalized to the
+ pre-defined dimensions. If input image is RGB it is first converted to the
+ gray-scale format.
+ The algorithm is identical to the following paper:
+ "On the Effectiveness of Local Binary Patterns in Face Anti-spoofing"
+
+ **Parameters:**
+
+ ``image`` : 2D or 3D :py:class:`numpy.ndarray`
+ Input image (RGB or gray-scale).
+
+ ``annotations`` : :py:class:`dict`
+ A dictionary containing annotations of the face bounding box.
+ Dictionary must be as follows ``{'topleft': (row, col), 'bottomright': (row, col)}``
+
+ ``face_size`` : :py:class:`int`
+ The size of the face after normalization.
+
+ **Returns:**
+
+ ``normbbx`` : 2D :py:class:`numpy.ndarray`
+ An image of the cropped face of the size (face_size, face_size).
+ """
+
+ if len(image.shape) == 3:
+
+ image = bob.ip.color.rgb_to_gray(image)
+
+ cutframe = image[annotations['topleft'][0]:annotations['bottomright'][0],
+ annotations['topleft'][1]:annotations['bottomright'][1]]
+
+ tempbbx = np.ndarray((face_size, face_size), 'float64')
+ normbbx = np.ndarray((face_size, face_size), 'uint8')
+ bob.ip.base.scale(cutframe, tempbbx) # normalization
+ tempbbx_ = tempbbx + 0.5
+ tempbbx_ = np.floor(tempbbx_)
+ normbbx = np.cast['uint8'](tempbbx_)
+
+ return normbbx
+
+
+ #==========================================================================
+ def __call__(self, image, annotations):
+ """
+ Call the ``normalize_image_size()`` method of this class.
+
+ **Parameters:**
+
+ ``image`` : 2D or 3D :py:class:`numpy.ndarray`
+ Input image (RGB or gray-scale).
+
+ ``annotations`` : :py:class:`dict`
+ A dictionary containing annotations of the face bounding box.
+ Dictionary must be as follows ``{'topleft': (row, col), 'bottomright': (row, col)}``
+
+ **Returns:**
+
+ ``norm_face_image`` : 2D :py:class:`numpy.ndarray`
+ An image of the cropped face of the size (self.face_size, self.face_size).
+ """
+
+ norm_face_image = self.normalize_image_size(image, annotations, self.face_size)
+
+ return norm_face_image
+
+
diff --git a/bob/pad/face/preprocessor/VideoFaceCrop.py b/bob/pad/face/preprocessor/VideoFaceCrop.py
index 2ef76572..48d50bb3 100644
--- a/bob/pad/face/preprocessor/VideoFaceCrop.py
+++ b/bob/pad/face/preprocessor/VideoFaceCrop.py
@@ -14,6 +14,10 @@ from bob.bio.face.preprocessor import FaceCrop
import bob.bio.video
+import numpy as np
+
+from bob.pad.face.preprocessor.ImageFaceCrop import ImageFaceCrop
+
#==============================================================================
# Main body:
@@ -54,9 +58,21 @@ class VideoFaceCrop(Preprocessor, object):
When run in parallel, the same random seed will be applied to all parallel processes.
Hence, results of parallel execution will differ from the results in serial execution.
+ ``check_face_size_flag`` : :py:class:`bool`
+ If True, only return the frames containing faces of the size above the
+ specified threshold ``min_face_size``. Default: False.
+
+ ``min_face_size`` : :py:class:`int`
+ The minimal size of the face in pixels. Only valid when ``check_face_size_flag``
+ is set to True. Default: 50.
+
+ ``use_local_cropper_flag`` : :py:class:`bool`
+ If True, use the local ImageFaceCrop class to crop faces in the frames.
+ Otherwise, the FaceCrop preprocessor from bob.bio.face is used.
+ Default: False.
+
``kwargs``
Remaining keyword parameters passed to the :py:class:`Base` constructor, such as ``color_channel`` or ``dtype``.
-
"""
#==========================================================================
@@ -67,15 +83,21 @@ class VideoFaceCrop(Preprocessor, object):
mask_sigma = None,
mask_neighbors = 5,
mask_seed = None,
+ check_face_size_flag = False,
+ min_face_size = 50,
+ use_local_cropper_flag = False,
**kwargs):
super(VideoFaceCrop, self).__init__(cropped_image_size = cropped_image_size,
- cropped_positions = cropped_positions,
- fixed_positions = fixed_positions,
- mask_sigma = mask_sigma,
- mask_neighbors = mask_neighbors,
- mask_seed = mask_seed,
- **kwargs)
+ cropped_positions = cropped_positions,
+ fixed_positions = fixed_positions,
+ mask_sigma = mask_sigma,
+ mask_neighbors = mask_neighbors,
+ mask_seed = mask_seed,
+ check_face_size_flag = check_face_size_flag,
+ min_face_size = min_face_size,
+ use_local_cropper_flag = use_local_cropper_flag,
+ **kwargs)
self.cropped_image_size = cropped_image_size
self.cropped_positions = cropped_positions
@@ -83,22 +105,75 @@ class VideoFaceCrop(Preprocessor, object):
self.mask_sigma = mask_sigma
self.mask_neighbors = mask_neighbors
self.mask_seed = mask_seed
+ self.check_face_size_flag = check_face_size_flag
+ self.min_face_size = min_face_size
+ self.use_local_cropper_flag = use_local_cropper_flag
# Save also the data stored in the kwargs:
for (k, v) in kwargs.items():
setattr(self, k, v)
- preprocessor = FaceCrop(cropped_image_size = cropped_image_size,
- cropped_positions = cropped_positions,
- fixed_positions = fixed_positions,
- mask_sigma = mask_sigma,
- mask_neighbors = mask_neighbors,
- mask_seed = mask_seed,
- **kwargs)
+ if self.use_local_cropper_flag:
+
+ preprocessor = ImageFaceCrop(face_size = self.cropped_image_size[0])
+
+ else:
+
+ preprocessor = FaceCrop(cropped_image_size = self.cropped_image_size,
+ cropped_positions = self.cropped_positions,
+ fixed_positions = self.fixed_positions,
+ mask_sigma = self.mask_sigma,
+ mask_neighbors = self.mask_neighbors,
+ mask_seed = self.mask_seed,
+ **kwargs)
self.video_preprocessor = bob.bio.video.preprocessor.Wrapper(preprocessor)
+ #==========================================================================
+ def check_face_size(self, frame_container, annotations, min_face_size):
+ """
+ Return the FrameContainer containing the frames with faces of the
+ size overcoming the specified threshold.
+
+ **Parameters:**
+
+ ``frame_container`` : FrameContainer
+ Video data stored in the FrameContainer, see ``bob.bio.video.utils.FrameContainer``
+ for further details.
+
+ ``annotations`` : :py:class:`dict`
+ A dictionary containing the annotations for each frame in the video.
+ Dictionary structure: ``annotations = {'1': frame1_dict, '2': frame1_dict, ...}``.
+ Where ``frameN_dict = {'topleft': (row, col), 'bottomright': (row, col)}``
+ is the dictionary defining the coordinates of the face bounding box in frame N.
+
+ ``min_face_size`` : :py:class:`int`
+ The minimal size of the face in pixels.
+ """
+
+ cleaned_frame_container = bob.bio.video.FrameContainer() # initialize the FrameContainer
+
+ selected_frame_idx = 0
+
+ for idx in range(0, len(annotations)): # idx - frame index
+
+ frame_annotations = annotations[str(idx)] # annotations for particular frame
+
+ # size of current face
+ face_size = np.min(np.array(frame_annotations['bottomright']) - np.array(frame_annotations['topleft']))
+
+ if face_size >= min_face_size: # check if face size is above the threshold
+
+ selected_frame = frame_container[idx][1] # get current frame
+
+ cleaned_frame_container.add(selected_frame_idx, selected_frame) # add current frame to FrameContainer
+
+ selected_frame_idx = selected_frame_idx + 1
+
+ return cleaned_frame_container
+
+
#==========================================================================
def __call__(self, frames, annotations):
"""
@@ -124,6 +199,10 @@ class VideoFaceCrop(Preprocessor, object):
preprocessed_video = self.video_preprocessor(frames = frames, annotations = annotations)
+ if self.check_face_size_flag:
+
+ preprocessed_video = self.check_face_size(preprocessed_video, annotations, self.min_face_size)
+
return preprocessed_video
diff --git a/bob/pad/face/preprocessor/__init__.py b/bob/pad/face/preprocessor/__init__.py
index 60586054..9f59b552 100644
--- a/bob/pad/face/preprocessor/__init__.py
+++ b/bob/pad/face/preprocessor/__init__.py
@@ -1,4 +1,5 @@
from .VideoFaceCrop import VideoFaceCrop
+from .ImageFaceCrop import ImageFaceCrop
def __appropriate__(*args):
@@ -21,5 +22,6 @@ def __appropriate__(*args):
__appropriate__(
VideoFaceCrop,
+ ImageFaceCrop,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
diff --git a/setup.py b/setup.py
index 791a68e8..8c0cf3dc 100644
--- a/setup.py
+++ b/setup.py
@@ -101,6 +101,8 @@ setup(
# registered preprocessors:
'bob.pad.preprocessor': [
'video-face-crop-preproc-64 = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64',
+ 'video-face-crop-preproc-64-face-50 = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64_face_50',
+ 'video-face-crop-preproc-64-face-50-local-cropper = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64_face_50_local_cropper',
],
# registered preprocessors:
@@ -112,6 +114,8 @@ setup(
'bob.pad.algorithm': [
'video-svm-pad-algorithm-10k-grid = bob.pad.face.config.algorithm.video_svm_pad_algorithm:video_svm_pad_algorithm_10k_grid',
'video-svm-pad-algorithm-10k-grid-mean-std = bob.pad.face.config.algorithm.video_svm_pad_algorithm:video_svm_pad_algorithm_10k_grid_mean_std',
+ 'video-svm-pad-algorithm-10k-grid-mean-std-frame-level = bob.pad.face.config.algorithm.video_svm_pad_algorithm:video_svm_pad_algorithm_10k_grid_mean_std_frame_level',
+ 'video-svm-pad-algorithm-default-svm-param-mean-std-frame-level = bob.pad.face.config.algorithm.video_svm_pad_algorithm:video_svm_pad_algorithm_default_svm_param_mean_std_frame_level',
],
# registered grid configurations:
--
GitLab