Commit 90dc144e authored by Guillaume HEUSCH's avatar Guillaume HEUSCH
Browse files

Initial commit: up to SVM preprocessing

parents
Pipeline #35754 failed with stages
in 6 minutes and 42 seconds
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
.coverage
record.txt
miniconda.sh
miniconda/
\ No newline at end of file
include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
\ No newline at end of file
This diff is collapsed.
include README.rst buildout.cfg COPYING version.txt requirements.txt
recursive-include doc *.rst *.png *.ico *.txt
\ No newline at end of file
.. -*- coding: utf-8 -*-
.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
:target: https://www.idiap.ch/software/bob/docs/bob/bob.paper.pad_mccnns_swirdiff/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/bob/docs/bob/bob.paper.pad_mccnns_swirdiff/master/index.html
.. image:: https://gitlab.idiap.ch/bob/bob.paper.pad_mccnns_swirdiff/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.paper.pad_mccnns_swirdiff/commits/master
.. image:: https://gitlab.idiap.ch/bob/bob.paper.pad_mccnns_swirdiff/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/bob/bob.paper.pad_mccnns_swirdiff/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.paper.pad_mccnns_swirdiff
.. image:: https://img.shields.io/pypi/v/bob.paper.pad_mccnns_swirdiff.svg
:target: https://pypi.python.org/pypi/bob.paper.pad_mccnns_swirdiff
================================================================================
Companion package for the paper on Deep CNNs and SWIR differences for Face PAD
================================================================================
This package is part of the signal-processing and machine learning toolbox Bob_.
It allows to reproduce the experiments presented in the following article::
@article{heusch-tbiom-2019,
author = {Guillaume Heusch, Anjith George, David Geissbuehler, Zoreh Mostaani and Sebastien Marcel},
title = {Deep Models and Shortwave Infrared Information to Detect Face Presentation Attacks},
journal = {IEEE Trans. on Biometrics, Behavior, and Identity Science},
volume = {XX}
issue = {YY}
year = {2020},
}
Installation
------------
Complete bob's `installation`_ instructions. Then, To get your environment up and running,
you should do the following:
.. code-block:: bash
$ git clone git@gitlab.idiap.ch:bob/bob.paper.pad_mccnns_swirdiff.git
$ cd bob.paper.pad_mccnns_swirdiff.git
$ conda env create -f environment.yml
$ conda activate pad_mccnns_swirdiff
$ buildout
Once all these commands have been executed, you can build the documentation:
.. code-block:: bash
$ ./bin/sphinx-build doc sphinx
And then, just open your favortie browser and go to ``sphinx/index.html``. You will
find all the steps to reproduce the experiments presented in the article.
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _installation: https://www.idiap.ch/software/bob/install
.. _mailing list: https://www.idiap.ch/software/bob/discuss
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import numpy
from bob.pad.base.algorithm import Algorithm
import bob.learn.libsvm
import bob.io.base
from bob.core.log import setup
logger = setup("bob.paper.facepad_swirdiff")
from operator import itemgetter
class SVM(Algorithm):
"""
This class interfaces an SVM classifier used specifically for pixel-based PAD.
Note that this algorithm is not generic at all, so beware !
Attributes
----------
rescale: bool
If you would like to rescale your data (to avoid a possible bug in lisvm sparse format)
gamma: float
The gamma in the RBF kernel
machine: :py:class:bob.learn.libsvm.Machine
The SVM machine
trainer: :py:class:bob.learn.libsvm.Trainer
The trainer
"""
def __init__(self, rescale=True, gamma=0.1, **kwargs):
""" Init function
Parameters
----------
rescale: bool
If you want your features to be rescaled
gamma: float
gamma value in the RBF kernel
performs_projection: bool
see super-class
requires_projector_training: bool
see super-class
"""
Algorithm.__init__(self,
performs_projection=True,
requires_projector_training=True,
**kwargs)
self.rescale = rescale
self.gamma = gamma
self.machine = None
self.trainer = bob.learn.libsvm.Trainer(machine_type='C_SVC', kernel_type='RBF', probability=True)
setattr(self.trainer, 'gamma', self.gamma)
def _rescale(self, features):
""" rescale the features between [-1 and 1]
Parameters
----------
features: numpy.ndarray
the features to be rescaled, in a 2-dimensional
array (n_features, features_dim)
Returns
-------
numpy.ndarray
The rescaled features, same size as original.
"""
for i in range(features.shape[0]):
min_value = numpy.min(features[i])
max_value = numpy.max(features[i])
features[i] = ((2 * (features[i] - min_value))/ (max_value - min_value)) - 1
return features
def train_projector(self, training_features, projector_file):
"""
Trains a SVM using positive and negative examples.
Note that the data (training_features) is provided as a list containing two
numpy arrays: the positive examples (labeled as '1')
and the negative examples (labeled as '-1').
In our specific case, the features are first reshaped.
Indeed, original features are of dimension 3 (n_frames, n_features, features_dim)
but the Trainer expects numpy arrays of two dimensions: (n_features, features_dim).
Parameters
----------
training_features: list of numpy.ndarray
see above
projector_file: str
The filename where to save the trained machine
"""
def reshape(features):
""" reshape features from 3 to 2 dimensions
Parameters
----------
features: numpy.ndarray
The features to be reshaped
Returns
-------
numpy.ndarray
The reshaped features
"""
feat = tuple([f.reshape(f.shape[0] * f.shape[1], f.shape[2]) for f in features])
return numpy.vstack(feat)
pos = reshape(training_features[0])
neg = reshape(training_features[1])
logger.debug("There are {} positive and {} negative examples".format(pos.shape[0], neg.shape[0]))
if self.rescale:
pos = self._rescale(pos)
neg = self._rescale(neg)
data = [pos, neg]
self.machine = self.trainer.train(data)
f = bob.io.base.HDF5File(projector_file, 'w')
self.machine.save(f)
def load_projector(self, projector_file):
""" loads the trained machine
Parameters
----------
projector_file: str
path to the saved SVM Machine
"""
f = bob.io.base.HDF5File(projector_file, 'r')
self.machine = bob.learn.libsvm.Machine(f)
def project(self, feature):
"""
Project the given feature.
Parameters
----------
feature: numpy.ndarray
Array of shape (n_frames, n_pixels, 6)
Returns
-------
float:
The probability for each pixels to be a positive example
"""
if self.rescale:
self._rescale(feature)
n_frames, n_pixels, _ = feature.shape
projected = numpy.zeros((n_frames, n_pixels))
for i in range(n_frames):
temp = self.machine.predict_class_and_probabilities(feature[i])
probabilities = temp[1]
projected[i] = probabilities[:, 0]
return projected
def score(self, toscore):
""" This function will not be used, since
a 3 dimensional array is expected
(n_frames, n_pixels, feature_dim)
"""
pass
def score_for_multiple_projections(self, toscore):
""" score all the frames of a given sequence
Parameters
----------
toscore: numpy.ndarray
The projected features, which are the probabilties
for each pixel to be of the positive class (i.e. skin-like)
Returns
-------
list
score for each frame (mean probability of pixels of the positive class)
"""
scores = []
for i in range(toscore.shape[0]):
scores.append(numpy.mean(toscore[i]))
return scores
#!/usr/bin/env python
# encoding: utf-8
import numpy
from bob.bio.base.extractor import Extractor
from bob.core.log import setup
logger = setup("bob.paper.facepad_swirdiff")
import bob.bio.video
from itertools import combinations
class SteinerExtractor(Extractor, object):
""" Compute spectral signature in the SWIR spectrum
The spectral signature, for each pixel, consists in a
combination of the differences between these wavelengths:
* 935nm
* 1060nm
* 1300nm
* 1550nm
The differences considered are given by :math:`d(a,b)` with the following constraint:
:math:`1 <= a < n-1` and :math:`a < b < n`. In our case of :math:`n = 4`, we hence have 6 differences:
* d[935, 1060]
* d[935, 1300]
* d[935, 1550]
* d[1060, 1300]
* d[1060, 1550]
* d[1300, 1550]
These features are described in the following article:
H. Steiner, S. Sporrer, A. Kold and N. Jung: "Design of an Active Multispectral
SWIR Camera System for Skin Detection and Face Verification", Journal of Sensors, 2015.
Note that the percentage of pixels that are retained could be specified.
This is done to avoid a too large number of training examples. Note however that
data are extracted for testing purposes (i.e. dev and eval), 100% of the pixels
should be retained
Attributes
----------
percent: int
The percentage of pixels that are retained.
debug: bool
If you would like to see some stuff
"""
def __init__(self, percent=100, debug=False, **kwargs):
""" Init function
Parameters
----------
percent: int
The percentage of pixels that are retained.
debug: bool
If you would like to see some stuff
"""
super(SteinerExtractor, self).__init__(**kwargs)
self.debug = debug
self.percent = percent
def __call__(self, data):
""" Compute the spectral signature
For each pixel, the spectral signature is computed.
When processing BATL2 data, we consider the following 6 channels:
* 1: 940 - 1050
* 5: 940 - 1300
* 9: 940 - 1550
* 15: 1050 - 1300
* 19: 1050 - 1550
* 33: 1300 - 1550
Parameters
----------
data: :py:class:`bob.bio.video.FrameContainer`
FrameContainer, where each "frame" contains 4 SWIR images
Returns
-------
numpy.ndarray
The spectral signature for a percentage of pixels in the image.
Each retainded pixel is considered as a feature.
"""
data_array = data.as_array()
data_array = data_array.astype('float')
n_frames, n_channels, height, width = data_array.shape
n_pixels_retained = int((self.percent / 100.0) * (height*width))
channels_list = list(range(n_channels))
diffs = list(combinations(channels_list, 2))
features = numpy.zeros((n_frames, n_pixels_retained, len(diffs)))
for i in range(n_frames):
for j, diff in enumerate(diffs):
temp = (data_array[i, diff[0]] - data_array[i, diff[1]]) / (data_array[i, diff[0]] + data_array[i, diff[1]] + 0.0001)
if self.debug:
from matplotlib import pyplot
pyplot.title("Frame {}, diff ({}, {})".format(i, diff[0], diff[1]))
pyplot.imshow(temp, cmap='gray')
pyplot.show()
temp = temp.flatten()
numpy.random.shuffle(temp)
features[i, :, j] = temp[:n_pixels_retained]
return features
def extract_with_skin_mask(self, data, annotations, attack=False):
""" Compute the spectral signature, using a skin-like distribution
on the pixels to figure out positives from negatives examples
Parameters
----------
data: :py:class:`bob.bio.video.FrameContainer`
FrameContainer, where each "frame" contains 4 SWIR images
annotations: numpy.ndarray
Boolean mask of shape (n_frames, height, width), specifying
which pixel in the image is considered as skin-like.
Returns
-------
"""
data_array = data.as_array()
n_frames, n_channels, height, width = data_array.shape
n_pixels_retained = int((self.percent / 100.0) * (height*width))
assert n_channels == 4, "4 channels are expected in your inputs"
channels_list = list(range(n_channels))
diffs = list(combinations(channels_list, 2))
features = numpy.zeros((n_frames, n_pixels_retained, len(diffs)))
for i in range(n_frames):
for j, diff in enumerate(diffs):
temp = (data_array[i, diff[0]] - data_array[i, diff[1]]) / (data_array[i, diff[0]] + data_array[i, diff[1]] + 0.0001)
if attack:
temp = temp[annotations[i] == 0]
else:
temp = temp[annotations[i] == 1]
temp = temp.flatten()
numpy.random.shuffle(temp)
try:
features[i, :, j] = temp[:n_pixels_retained]
except IndexError:
features[i, :, j] = temp
return features
#!/usr/bin/env python
# encoding: utf-8
import numpy
from bob.core.log import setup
logger = setup("bob.paper.facepad_swirdiff")
import bob.bio.video
from bob.bio.base.preprocessor import Preprocessor
from bob.bio.face.preprocessor import FaceCrop
from bob.bio.face.preprocessor import FaceDetect
class SteinerPreprocessorBRSU(Preprocessor):
"""
This class is used to preprocess multispectral data
It will crop the faces, and resize them, in several channels:
* color
* SWIR 940nm
* SWIR 1050nm
* SWIR 1300nm
* SWIR 1550nm
Attributes
----------
face_cropper: :py:class:`bob.bio.face.preprocessor.FaceCrop`
The face cropper
cropped_height: int
The desired height of the cropped face
cropped_width: int
The desired width of the cropped face
debug: bool
If you want to see what's going on
"""
def __init__(self, face_size=128, debug=False, save_gray=False, **kwargs):
"""
Init function
Parameters
----------
face_size: int
The size of the cropped faces (square)
asbolute: bool
If you want to compute the absolute difference
debug: bool
If you want to see what's going on
save_gray: bool
If you want to save the grayscale face
"""
super(SteinerPreprocessorBRSU, self).__init__(**kwargs)
self.debug = debug
self.save_gray = save_gray
# cropped size
self.cropped_height = face_size
self.cropped_width = face_size
# original (from bob.bio.face), should yield a relatively tight crop
right_eye_pos = (self.cropped_height // 5, self.cropped_width // 4 - 1)
left_eye_pos = (self.cropped_height // 5, self.cropped_width // 4 * 3)
# define the face cropper
self.face_cropper = FaceCrop(
cropped_image_size=(self.cropped_height, self.cropped_width),
cropped_positions={'leye': left_eye_pos, 'reye': right_eye_pos})
# the face detector
self.face_detector = FaceDetect(face_cropper=self.face_cropper, use_flandmark = False)
def __call__(self, images, annotations=None):
"""
Does the actual preprocessing.
Face detection in the grayscale image, and in one of the SWIR channels.
Faces are then cropped, resized and differences are computed for SWIR channels.
Parameters
----------
images: dict
The dictionary with the face images. The key is the modality
annotations: dict
The annotations (usually the eyes position)
Returns
-------
:py:class:`bob.bio.video.FrameContainer`
The face in grayscale and all the SWIR differences
"""
faces = {}
annotations = {}
faces['color'] = self.face_detector(images['color'])
# now find the best face in different SWIR images, to be used as reference
best_quality = 0
for mod in images.keys():
if 'nm' in mod:
bbox, quality = bob.ip.facedetect.detect_single_face(images[mod])
if quality > best_quality:
best_eyes = bob.ip.facedetect.expected_eye_positions(bbox)
# now crop the faces in SWIR images
for mod in images.keys():
if 'nm' in mod:
faces[mod] = self.face_cropper(images[mod], best_eyes)
if not self.save_gray:
del faces['color']
if self.debug:
from matplotlib import pyplot
f, ax = pyplot.subplots(len(faces.keys()), 2)
counter = 0
for mod in faces.keys():
ax[counter][0].imshow(bob.io.image.to_matplotlib(images[mod]))
ax[counter][1].imshow(faces[mod], cmap='gray')
counter += 1
pyplot.show()
# build the final result - a frame container is expected ...
final_array = numpy.zeros((len(faces), self.cropped_height, self.cropped_width), dtype='uint8')
for i, mod in enumerate(faces.keys()):
final_array[i] = faces[mod].astype('uint8')
fc = bob.bio.video.FrameContainer()
fc.add('0', final_array)
return fc
class SteinerPreprocessor(Preprocessor):
"""
This class is used to preprocess multispectral data
It will crop the faces, and resize them, in several channels:
* color
* SWIR 940nm
* SWIR 1050nm
* SWIR 1300nm
* SWIR 1550nm
Attributes
----------
face_cropper: :py:class:`bob.bio.face.preprocessor.FaceCrop`
The face cropper
cropped_height: int