Merge branch '23-follow-up-from-wip-rppg-as-features-for-pad' into 'master'

Resolve "Follow-up from "WIP: rPPG as features for PAD""

Closes #23

See merge request !69

bob/pad/face/database/maskattack.py

-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-
-import os
-import numpy as np
-import bob.io.video
-from bob.bio.video import FrameSelector, FrameContainer
-from bob.pad.face.database import VideoPadFile  # Used in MsuMfsdPadFile class
-from bob.pad.base.database import PadDatabase
-
-class MaskAttackPadFile(VideoPadFile):
-    """
-    A high level implementation of the File class for the 3DMAD database.
-    """
-
-    def __init__(self, f):
-        """
-        **Parameters:**
-
-        ``f`` : :py:class:`object`
-            An instance of the File class defined in the low level db interface
-            of the 3DMAD database, in the bob.db.maskattack.models.py file.
-        """
-
-        self.f = f
-        if f.is_real():
-            attack_type = None
-        else:
-            attack_type = 'mask'
-
-        super(MaskAttackPadFile, self).__init__(
-            client_id=f.client_id,
-            path=f.path,
-            attack_type=attack_type,
-            file_id=f.id)
-
-    #==========================================================================
-    def load(self, directory=None, extension='.avi', frame_selector=FrameSelector(selection_style='all')):
-        """
-        Overridden version of the load method defined in the ``VideoPadFile``.
-
-        **Parameters:**
-
-        ``directory`` : :py:class:`str`
-            String containing the path to the MSU MFSD database.
-            Default: None
-
-        ``extension`` : :py:class:`str`
-            Extension of the video files in the MSU MFSD database.
-            Note: ``extension`` value is not used in the code of this method.
-            Default: None
-
-        ``frame_selector`` : ``FrameSelector``
-            The frame selector to use.
-
-        **Returns:**
-
-        ``video_data`` : FrameContainer
-            Video data stored in the FrameContainer, see ``bob.bio.video.utils.FrameContainer``
-            for further details.
-        """
-        vfilename = self.make_path(directory, extension)
-        video = bob.io.video.reader(vfilename)
-        video_data_array = video.load()
-        
-        return frame_selector(video_data_array)
-
-
-#==============================================================================
-class MaskAttackPadDatabase(PadDatabase):
-    """
-    A high level implementation of the Database class for the 3DMAD database.
-    """
-
-    def __init__(
-            self,
-            protocol=None,  
-            original_directory=None,
-            original_extension='.avi',
-            **kwargs):
-        """
-        **Parameters:**
-
-        ``protocol`` : :py:class:`str` or ``None``
-            The name of the protocol that defines the default experimental setup for this database.
-
-        ``original_directory`` : :py:class:`str`
-            The directory where the original data of the database are stored.
-
-        ``original_extension`` : :py:class:`str`
-            The file name extension of the original data.
-
-        ``kwargs``
-            The arguments of the :py:class:`bob.bio.base.database.BioDatabase` base class constructor.
-        """
-
-        from bob.db.maskattack import Database as LowLevelDatabase
-
-        self.db = LowLevelDatabase()
-
-        # Since the high level API expects different group names than what the low
-        # level API offers, you need to convert them when necessary
-        self.low_level_group_names = (
-            'world', 'dev',
-            'test')  # group names in the low-level database interface
-        self.high_level_group_names = (
-            'train', 'dev',
-            'eval')  # names are expected to be like that in objects() function
-
-        # Always use super to call parent class methods.
-        super(MaskAttackPadDatabase, self).__init__(
-            name='maskattack',
-            protocol=protocol,
-            original_directory=original_directory,
-            original_extension=original_extension,
-            **kwargs)
-
-    @property
-    def original_directory(self):
-        return self.db.original_directory
-
-    @original_directory.setter
-    def original_directory(self, value):
-        self.db.original_directory = value
-
-    #==========================================================================
-    def objects(self,
-                groups=None,
-                protocol=None,
-                purposes=None,
-                model_ids=None,
-                **kwargs):
-        """
-        This function returns lists of MaskAttackPadFile objects, which fulfill the given restrictions.
-
-        Keyword parameters:
-
-        ``groups`` : :py:class:`str`
-            OR a list of strings.
-            The groups of which the clients should be returned.
-            Usually, groups are one or more elements of ('train', 'dev', 'eval')
-
-        ``protocol`` : :py:class:`str`
-            The protocol for which the clients should be retrieved.
-            Note: this argument is not used in the code, because ``objects`` method of the
-            low-level BD interface of the MSU MFSD doesn't have ``protocol`` argument.
-
-        ``purposes`` : :py:class:`str`
-            OR a list of strings.
-            The purposes for which File objects should be retrieved.
-            Usually it is either 'real' or 'attack'.
-
-        ``model_ids``
-            This parameter is not supported in PAD databases yet.
-
-        **Returns:**
-
-        ``files`` : [MsuMfsdPadFile]
-            A list of MsuMfsdPadFile objects.
-        """
-
-        # Convert group names to low-level group names here.
-        groups = self.convert_names_to_lowlevel(
-            groups, self.low_level_group_names, self.high_level_group_names)
-        # Since this database was designed for PAD experiments, nothing special
-        # needs to be done here.
-
-        print("Objects method called with groups = {}, protocol = {}, purposes = {}, model_ids = {}".format(groups, protocol, purposes, model_ids))
-        #print("Kwargs -> {}".format(**kwargs))
-        #print("Translated groups = {}".frima)
-        
-        # for training
-
-        # for dev
-
-        # for eval
-        lowlevel_purposes = []
-        if purposes == 'real':
-          lowlevel_purposes = ['trainReal', 'probeReal', 'classifyReal']
-        else:
-          lowlevel_purposes = ['trainMask', 'probeMask', 'classifyMask']
-          
-        #if groups == ['world']:
-        #    lowlevel_purposes = ['trainMask']
-        #  if groups == ['world']:
-        #    lowlevel_purposes = ['trainMask']
-        #print(lowlevel_purposes)
-        files = self.db.objects(sets=groups, purposes=lowlevel_purposes, **kwargs)
-
-        files = [MaskAttackPadFile(f) for f in files]
-
-        return files
-
-    #==========================================================================
-    def annotations(self, file):
-        """
-        Return annotations for a given file object ``f``, which is an instance
-        of ``MsuMfsdPadFile`` defined in the HLDI of the MSU MFSD DB.
-        The ``load()`` method of ``MsuMfsdPadFile`` class (see above)
-        returns a video, therefore this method returns bounding-box annotations
-        for each video frame. The annotations are returned as dictionary of dictionaries.
-
-        **Parameters:**
-
-        ``f`` : :py:class:`object`
-            An instance of ``MsuMfsdPadFile`` defined above.
-
-        **Returns:**
-
-        ``annotations`` : :py:class:`dict`
-            A dictionary containing the annotations for each frame in the video.
-            Dictionary structure: ``annotations = {'1': frame1_dict, '2': frame1_dict, ...}``.
-            Where ``frameN_dict = {'topleft': (row, col), 'bottomright': (row, col)}``
-            is the dictionary defining the coordinates of the face bounding box in frame N.
-        """
-        return None
-
-        #annots = f.f.bbx(
-        #    directory=self.original_directory
-        #)  # numpy array containing the face bounding box data for each video frame, returned data format described in the f.bbx() method of the low level interface
-
-        #annotations = {}  # dictionary to return
-
-        #for frame_annots in annots:
-
-        #    topleft = (np.int(frame_annots[2]), np.int(frame_annots[1]))
-        #    bottomright = (np.int(frame_annots[2] + frame_annots[4]),
-        #                   np.int(frame_annots[1] + frame_annots[3]))
-
-        #    annotations[str(np.int(frame_annots[0]))] = {
-        #        'topleft': topleft,
-        #        'bottomright': bottomright
-        #    }
-
-        #return annotations
-
-    #def model_with_ids_protocol(groups=None, protocol=None):
-    #  pass

bob/pad/face/extractor/LTSS.py

@@ -54,9 +54,12 @@ class LTSS(Extractor, object):
       The length of the signal to consider (in seconds)
     
     """
-    super(LTSS, self).__init__()
+    super(LTSS, self).__init__(**kwargs)
     self.framerate = framerate
+    
+    # TODO: try to use window size as NFFT - Guillaume HEUSCH, 04-07-2018
     self.nfft = nfft
+    
     self.debug = debug
     self.window_size = window_size
     self.concat = concat
@@ -80,27 +83,40 @@ class LTSS(Extractor, object):
 
     # log-magnitude of DFT coefficients
     log_mags = []
-   
+
     # go through windows
     for w in range(0, (signal.shape[0] - self.window_size), window_stride):
+      
+      # n is even, as a consequence the fft is as follows [y(0), Re(y(1)), Im(y(1)), ..., Re(y(n/2))]
+      # i.e. each coefficient, except first and last, is represented by two numbers (real + imaginary)
       fft = rfft(signal[w:w+self.window_size], n=self.nfft)
-      mags = numpy.zeros(int(self.nfft/2), dtype=numpy.float64)
       
-      # XXX : bug was here (no clipping)
+      # the magnitude is the norm of the complex numbers, so its size is n/2 + 1
+      mags = numpy.zeros((int(self.nfft/2) + 1), dtype=numpy.float64)
+      
+      # first coeff is real
       if abs(fft[0]) < 1:
         mags[0] = 1
       else:
         mags[0] = abs(fft[0])
-      # XXX 
 
+      # go through coeffs 2 to n/2
       index = 1
       for i in range(1, (fft.shape[0]-1), 2):
         mags[index] = numpy.sqrt(fft[i]**2 + fft[i+1]**2)
         if mags[index] < 1:
           mags[index] = 1
         index += 1
+      
+      # last coeff is real too
+      if abs(fft[-1]) < 1:
+        mags[index] = 1
+      else:
+        mags[index] = abs(fft[-1])
+      
       log_mags.append(numpy.log(mags))
 
+    # build final feature
     log_mags = numpy.array(log_mags)
     mean = numpy.mean(log_mags, axis=0)
     std = numpy.std(log_mags, axis=0)

bob/pad/face/extractor/LiFeatures.py → bob/pad/face/extractor/LiSpectralFeatures.py

@@ -9,7 +9,7 @@ from bob.core.log import setup
 logger = setup("bob.pad.face")
 
 
-class LiFeatures(Extractor, object):
+class LiSpectralFeatures(Extractor, object):
   """Compute features from pulse signals in the three color channels.
 
   The features are described in the following article:
@@ -41,7 +41,7 @@ class LiFeatures(Extractor, object):
       Plot stuff
   
     """
-    super(LiFeatures, self).__init__()
+    super(LiSpectralFeatures, self).__init__()
     self.framerate = framerate
     self.nfft = nfft
     self.debug = debug

bob/pad/face/extractor/PPGSecure.py

@@ -72,13 +72,13 @@ class PPGSecure(Extractor, object):
     # get the frequencies
     f = numpy.fft.fftfreq(self.nfft) * self.framerate
    
-    # we have 5x3 pulse signals, in different regions across 3 channels
+    # we have 5 pulse signals, in different regions 
     ffts = numpy.zeros((5, output_dim))
     for i in range(5):
       ffts[i] = abs(numpy.fft.rfft(signal[:, i], n=self.nfft))
 
     fft = numpy.concatenate([ffts[0], ffts[1], ffts[2], ffts[3], ffts[4]])
-      
+
     if self.debug: 
       from matplotlib import pyplot
       pyplot.plot(range(output_dim*5), fft, 'k')

bob/pad/face/extractor/__init__.py

@@ -2,7 +2,7 @@ from .LBPHistogram import LBPHistogram
 from .ImageQualityMeasure import ImageQualityMeasure
 from .FrameDiffFeatures import FrameDiffFeatures
 
-from .LiFeatures import LiFeatures 
+from .LiSpectralFeatures import LiSpectralFeatures 
 from .LTSS import LTSS 
 from .PPGSecure import PPGSecure 
 
@@ -28,5 +28,8 @@ __appropriate__(
     LBPHistogram,
     ImageQualityMeasure,
     FrameDiffFeatures,
+    LiSpectralFeatures,
+    LTSS,
+    PPGSecure,
 )
 __all__ = [_ for _ in dir() if not _.startswith('_')]

bob/pad/face/preprocessor/Chrom.py

@@ -81,7 +81,7 @@ class Chrom(Preprocessor, object):
     self.debug = debug
     self.skin_filter = bob.ip.skincolorfilter.SkinColorFilter()
 
-  def __call__(self, frames, annotations):
+  def __call__(self, frames, annotations=None):
     """Computes the pulse signal for the given frame sequence
 
     Parameters

bob/pad/face/preprocessor/Li.py → bob/pad/face/preprocessor/LiPulseExtraction.py

@@ -17,10 +17,15 @@ from bob.rppg.cvpr14.filter_utils import detrend
 from bob.rppg.cvpr14.filter_utils import average
 
 
-class Li(Preprocessor):
+class LiPulseExtraction(Preprocessor):
   """Extract pulse signal from a video sequence.
   
-  The pulse is extracted according to Li's CVPR 14 algorithm.
+  The pulse is extracted according to a simplified version of Li's CVPR 14 algorithm.
+
+  It is described in:
+  X. Li, J, Komulainen, G. Zhao, P-C Yuen and M. Pietikäinen
+  "Generalized face anti-spoofing by detecting pulse from face videos"
+  Intl Conf on Pattern Recognition (ICPR), 2016
 
   See the documentation of `bob.rppg.base`
 
@@ -65,7 +70,7 @@ class Li(Preprocessor):
       Plot some stuff 
 
     """
-    super(Li, self).__init__(**kwargs)
+    super(LiPulseExtraction, self).__init__(**kwargs)
     self.indent = indent
     self.lambda_ = lambda_
     self.window = window
@@ -148,6 +153,8 @@ class Li(Preprocessor):
 
       ldms = numpy.array(ldms)
       mask_points, mask = kp66_to_mask(frame, ldms, self.indent, self.debug)
+      
+      #XXX : be sure that the 3 colors are returned !!
       face_color[i] = compute_average_colors_mask(frame, mask, self.debug)
 
       previous_ldms = ldms 

bob/pad/face/preprocessor/PPGSecure.py

@@ -130,7 +130,7 @@ class PPGSecure(Preprocessor):
       # get the mask and the green value in the different ROIs
       masks = self._get_masks(frame, ldms)
       for k in range(5):
-        green_mean[i, k] = compute_average_colors_mask(frame, masks[k], self.debug)[1]
+        green_mean[i, k] = compute_average_colors_mask(frame, masks[k], self.debug)
 
       previous_ldms = ldms 
 

bob/pad/face/preprocessor/__init__.py

@@ -2,7 +2,7 @@ from .FaceCropAlign import FaceCropAlign
 from .FrameDifference import FrameDifference
 from .VideoSparseCoding import VideoSparseCoding
 
-from .Li import Li
+from .LiPulseExtraction import LiPulseExtraction
 from .Chrom import Chrom
 from .SSR import SSR
 from .PPGSecure import PPGSecure
@@ -29,5 +29,9 @@ __appropriate__(
     FaceCropAlign,
     FrameDifference,
     VideoSparseCoding,
+    LiPulseExtraction,
+    Chrom,
+    SSR,
+    PPGSecure,
 )
 __all__ = [_ for _ in dir() if not _.startswith('_')]

bob/pad/face/test/test.py

@@ -28,7 +28,15 @@ from ..extractor import LBPHistogram
 
 from ..extractor import ImageQualityMeasure
 
-import random
+from ..preprocessor import LiPulseExtraction
+from ..preprocessor import Chrom
+from ..preprocessor import PPGSecure as PPGPreprocessor
+from ..preprocessor import SSR
+
+from ..extractor import LTSS
+from ..extractor import LiSpectralFeatures
+from ..extractor import PPGSecure as PPGExtractor
+
 
 from ..preprocessor.FaceCropAlign import detect_face_landmarks_in_image
 
@@ -371,3 +379,99 @@ def convert_array_to_list_of_frame_cont(data):
             frame_container)  # add current frame to FrameContainer
 
     return frame_container_list
+
+
+def test_preprocessor_LiPulseExtraction():
+      """ Test the pulse extraction using Li's ICPR 2016 algorithm.
+      """
+
+      image = load(datafile('test_image.png', 'bob.pad.face.test'))
+      annotations = {'topleft': (95, 155), 'bottomright': (215, 265)}
+      video, annotations = convert_image_to_video_data(image, annotations, 100)
+      
+      preprocessor = LiPulseExtraction(debug=False)
+      pulse = preprocessor(video, annotations)
+      assert pulse.shape == (100, 3)
+
+
+def test_preprocessor_Chrom():
+      """ Test the pulse extraction using CHROM algorithm.
+      """
+
+      image = load(datafile('test_image.png', 'bob.pad.face.test'))
+      annotations = {'topleft': (95, 155), 'bottomright': (215, 265)}
+      video, annotations = convert_image_to_video_data(image, annotations, 100)
+  
+      preprocessor = Chrom(debug=False)
+      pulse = preprocessor(video, annotations)
+      assert pulse.shape[0] == 100
+
+
+def test_preprocessor_PPGSecure():
+      """ Test the pulse extraction using PPGSecure algorithm.
+      """
+
+      image = load(datafile('test_image.png', 'bob.pad.face.test'))
+      annotations = {'topleft': (456, 212), 'bottomright': (770, 500)}
+      video, annotations = convert_image_to_video_data(image, annotations, 100)
+  
+      preprocessor = PPGPreprocessor(debug=False)
+      pulse = preprocessor(video, annotations)
+      assert pulse.shape == (100, 5)
+
+
+def test_preprocessor_SSR():
+      """ Test the pulse extraction using SSR algorithm.
+      """
+      
+      image = load(datafile('test_image.png', 'bob.pad.face.test'))
+      annotations = {'topleft': (95, 155), 'bottomright': (215, 265)}
+      video, annotations = convert_image_to_video_data(image, annotations, 100)
+  
+      preprocessor = SSR(debug=False)
+      pulse = preprocessor(video, annotations)
+      assert pulse.shape[0] == 100
+
+
+def test_extractor_LTSS():
+      """ Test Long Term Spectrum Statistics (LTSS) Feature Extractor 
+      """
+      
+      # "pulse" in 3 color channels
+      data = np.random.random((200, 3))
+      
+      extractor = LTSS(concat=True)
+      features = extractor(data)
+      # n = number of FFT coefficients (default is 64)
+      # (n/2 + 1) * 2 (mean and std) * 3 (colors channels)
+      assert features.shape[0] == 33*2*3
+      
+      extractor = LTSS(concat=False)
+      features = extractor(data)
+      # only one "channel" is considered
+      assert features.shape[0] == 33*2
+
+
+def test_extractor_LiSpectralFeatures():
+      """ Test Li's ICPR 2016 Spectral Feature Extractor 
+      """
+      
+      # "pulse" in 3 color channels
+      data = np.random.random((200, 3))
+      
+      extractor = LiSpectralFeatures()
+      features = extractor(data)
+      assert features.shape[0] == 6 
+     
+
+def test_extractor_PPGSecure():
+      """ Test PPGSecure Spectral Feature Extractor 
+      """
+      # 5 "pulses" 
+      data = np.random.random((200, 5))
+      
+      extractor = PPGExtractor()
+      features = extractor(data)
+      # n = number of FFT coefficients (default is 32)
+      # 5 (pulse signals) * (n/2 + 1)
+      assert features.shape[0] == 5*17

conda/meta.yaml

@@ -35,14 +35,13 @@ requirements:
     - bob.learn.libsvm
     - bob.ip.dlib
     - bob.ip.facelandmarks
-    - bob.rppg.base
+    - bob.rppg.base >=2.0.0
   run:
     - python
     - setuptools
     - six
     - numpy >=1.11 
     - scikit-learn
-    - bob.rppg.base
 
 test:
   imports:

doc/index.rst

@@ -22,6 +22,7 @@ Users Guide
    installation
    baselines
    other_pad_algorithms
+   pulse
    references
    resources
    api

doc/pulse.rst

+
+.. _bob.pad.face.pulse:
+
+===============
+Pulse-based PAD
+===============
+
+In this section, we briefly describe our work made for face
+presentation attack detection using the blood volume pulse,
+inferred from remote photoplesthymograpy.
+
+The basic idea here is to retrieve the pulse signals from
+face video sequences, to derive features from their frequency
+spectrum and then to learn a classifier to discriminate
+between *bonafide* attempts from presentation attacks.
+
+For this purpose, we describe both :py:class:`bob.bio.base.preprocessor.Preprocessor` and
+:py:class:`bob.bio.base.extractor.Extractor` specifically dedicated to this task.
+
+Preprocessors: Pulse Extraction
+-------------------------------
+
+Preprocessors basically extract pulse signals from face video 
+sequences. They heavily rely on what has been done in `bob.rppg.base`
+so you may want to have a look at `its documentation <https://www.idiap.ch/software/bob/docs/bob/bob.rppg.base/master/index.html>`_. 
+
+In this package, 4 preprocessors have been implemented:
+
+  1. :py:class:`bob.pad.face.preprocessor.LiPulseExtraction` described in [Li_ICPR_2016]_.
+  
+  2. :py:class:`bob.pad.face.preprocessor.Chrom` described in [CHROM]_.
+
+  3. :py:class:`bob.pad.face.preprocessor.SSR` described in [SSR]_.
+
+  4. :py:class:`bob.pad.face.preprocessor.PPGSecure` described in [NOWARA]_.
+
+
+Extractors: Features from Pulses
+--------------------------------
+
+Extractors compute and retrieve features from the pulse signal. All
+implemented extractors act on the frequency spectrum of the pulse signal.
+
+In this package, 3 extractors have been implemented:
+
+  1. :py:class:`bob.pad.face.extractor.LiSpectralFeatures` described in [Li_ICPR_2016]_.
+  
+  2. :py:class:`bob.pad.face.extractor.PPGSecure` described in [NOWARA]_.
+
+  3. :py:class:`bob.pad.face.extractor.LTSS` described in [LTSS]_.
+
+
+
+References
+----------
+
+
+.. [Li_ICPR_2016] *X. Li, J, Komulainen, G. Zhao, P-C Yuen and M. Pietikäinen*
+  **Generalized face anti-spoofing by detecting pulse from face videos**,
+  Intl Conf on Pattern Recognition (ICPR), 2016
+
+.. [CHROM] *de Haan, G. & Jeanne, V*. **Robust Pulse Rate from Chrominance based rPPG**, IEEE Transactions on Biomedical Engineering, 2013. `pdf <http://www.es.ele.tue.nl/~dehaan/pdf/169_ChrominanceBasedPPG.pdf>`__
+
+.. [SSR] *Wang, W., Stuijk, S. and de Haan, G*. **A Novel Algorithm for Remote Photoplesthymograpy: Spatial Subspace Rotation**, IEEE Trans. On Biomedical Engineering, 2015
+
+.. [NOWARA] *E. M. Nowara, A. Sabharwal, A. Veeraraghavan*. **PPGSecure: Biometric Presentation Attack Detection Using Photopletysmograms**, IEEE International Conference on Automatic Face & Gesture Recognition, 2017
+
+.. [LTSS] *H .Muckenhirn, P. Korshunov, M. Magimai-Doss, S Marcel*. **Long-Term Spectral Statistics for Voice Presentation Attack Detection**, IEEE Trans. On Audio, Speech and Language Processing, 2017

requirements.txt

@@ -16,4 +16,4 @@ bob.ip.facelandmarks
 bob.learn.libsvm
 bob.learn.linear
 scikit-learn
-bob.rppg.base
+bob.rppg.base >= 2.0.0