From 2cd2c73e749464c33a798d44b1259dcc56f1d9a9 Mon Sep 17 00:00:00 2001
From: Olegs NIKISINS <onikisins@italix03.idiap.ch>
Date: Thu, 1 Jun 2017 16:57:01 +0200
Subject: [PATCH] Added VideoQualityMeasure extractor which is based on
ImageQualityMeasure and VideoDataLoader classes
---
.../config/extractor/video_quality_measure.py | 15 ++
bob/pad/face/config/preprocessor/filename.py | 5 +
bob/pad/face/database/replay.py | 11 +-
bob/pad/face/extractor/ImageQualityMeasure.py | 114 ++++++++++++++
bob/pad/face/extractor/VideoDataLoader.py | 117 ++++++++++++++
bob/pad/face/extractor/VideoLBPHistogram.py | 2 +-
bob/pad/face/extractor/VideoQualityMeasure.py | 144 ++++++++++++++++++
bob/pad/face/extractor/__init__.py | 7 +-
setup.py | 2 +
9 files changed, 412 insertions(+), 5 deletions(-)
create mode 100644 bob/pad/face/config/extractor/video_quality_measure.py
create mode 100644 bob/pad/face/config/preprocessor/filename.py
create mode 100644 bob/pad/face/extractor/ImageQualityMeasure.py
create mode 100644 bob/pad/face/extractor/VideoDataLoader.py
create mode 100644 bob/pad/face/extractor/VideoQualityMeasure.py
diff --git a/bob/pad/face/config/extractor/video_quality_measure.py b/bob/pad/face/config/extractor/video_quality_measure.py
new file mode 100644
index 00000000..f6910cc8
--- /dev/null
+++ b/bob/pad/face/config/extractor/video_quality_measure.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+from bob.pad.face.extractor import VideoQualityMeasure
+
+
+#=======================================================================================
+# Define instances here:
+
+galbally=True
+msu=True
+dtype=None
+
+video_quality_measure_galbally_msu = VideoQualityMeasure(galbally=galbally,
+ msu=msu,
+ dtype=dtype)
diff --git a/bob/pad/face/config/preprocessor/filename.py b/bob/pad/face/config/preprocessor/filename.py
new file mode 100644
index 00000000..feeefff0
--- /dev/null
+++ b/bob/pad/face/config/preprocessor/filename.py
@@ -0,0 +1,5 @@
+from bob.bio.base.preprocessor import Filename
+
+# This preprocessor does nothing, returning just the name of the file to extract the features from:
+empty_preprocessor = Filename()
+
diff --git a/bob/pad/face/database/replay.py b/bob/pad/face/database/replay.py
index 6a1a5336..e8f052f2 100644
--- a/bob/pad/face/database/replay.py
+++ b/bob/pad/face/database/replay.py
@@ -66,9 +66,9 @@ class ReplayPadFile(PadFile):
**Returns:**
- ``filtered_image`` : :py:class:`dict`
- A dictionary containing the key-value pairs: "video" key containing the frames data,
- and "bbx" containing the coordinates of the face bounding boxes for each frame.
+ ``video_data`` : FrameContainer
+ Video data stored in the FrameContainer, see ``bob.bio.video.utils.FrameContainer``
+ for further details.
"""
path = self.f.make_path(directory=directory, extension=extension) # path to the video file
@@ -150,6 +150,11 @@ class ReplayPadDatabase(PadDatabase):
``model_ids``
This parameter is not supported in PAD databases yet
+
+ **Returns:**
+
+ ``files`` : :py:class:`str`
+ A list of ReplayPadFile objects.
"""
# Convert group names to low-level group names here.
groups = self.convert_names_to_lowlevel(groups, self.low_level_group_names, self.high_level_group_names)
diff --git a/bob/pad/face/extractor/ImageQualityMeasure.py b/bob/pad/face/extractor/ImageQualityMeasure.py
new file mode 100644
index 00000000..cec240ef
--- /dev/null
+++ b/bob/pad/face/extractor/ImageQualityMeasure.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+#==============================================================================
+# Import what is needed here:
+
+from __future__ import division
+from bob.bio.base.extractor import Extractor
+from bob.ip.qualitymeasure import galbally_iqm_features as iqm
+from bob.ip.qualitymeasure import msu_iqa_features as iqa
+import numpy
+import logging
+
+logger = logging.getLogger(__name__)
+
+#==============================================================================
+# Main body:
+
+class ImageQualityMeasure(Extractor):
+ """
+ This class is designed to extract Image Quality Measures given input RGB
+ image. For further documentation and description of features,
+ see :ref:`bob.ip.qualitymeasure`.
+
+ **Parameters:**
+
+ ``galbally`` : :py:class:`bool`
+ If ``True``, galbally features will be added to the features.
+ Default: ``True``.
+
+ ``msu`` : :py:class:`bool`
+ If ``True``, MSU features will be added to the features.
+ Default: ``True``.
+
+ ``dtype`` : numpy.dtype
+ The data type of the resulting feature vector.
+ Default: ``None``.
+ """
+ #==========================================================================
+ def __init__(self,
+ galbally=True,
+ msu=True,
+ dtype=None,
+ **kwargs):
+
+ Extractor.__init__(self,
+ galbally=galbally,
+ msu=msu,
+ dtype=dtype,
+ **kwargs)
+
+ self.dtype = dtype
+ self.galbally = galbally
+ self.msu = msu
+
+
+ #==========================================================================
+ def __call__(self, data):
+ """
+ Compute Image Quality Measures given input RGB image.
+
+ **Parameters:**
+
+ ``data`` : 3D :py:class:`numpy.ndarray`
+ Input RGB image of the dimensionality (3, Row, Col), as returned
+ by Bob image loading routines.
+
+ **Returns:**
+
+ ``features`` : 1D :py:class:`numpy.ndarray`
+ Feature vector containing Image Quality Measures.
+ """
+
+ assert isinstance(data, numpy.ndarray)
+ assert self.galbally or self.msu
+
+ features = []
+
+ if self.galbally:
+
+ try:
+
+ gf_set = iqm.compute_quality_features(data)
+ gf_set = numpy.nan_to_num(gf_set)
+ features = numpy.hstack((features, gf_set))
+
+ except Exception as e:
+
+ logger.error(
+ "Failed to extract galbally features.", exc_info=e)
+
+ return None
+
+ if self.msu:
+
+ try:
+
+ msuf_set = iqa.compute_msu_iqa_features(data)
+ msuf_set = numpy.nan_to_num(msuf_set)
+ features = numpy.hstack((features, msuf_set))
+
+ except Exception as e:
+
+ logger.error("Failed to extract MSU features.", exc_info=e)
+
+ return None
+
+ elif self.dtype is not None:
+
+ features = features.astype(self.dtype)
+
+ return features
+
+
diff --git a/bob/pad/face/extractor/VideoDataLoader.py b/bob/pad/face/extractor/VideoDataLoader.py
new file mode 100644
index 00000000..18a314db
--- /dev/null
+++ b/bob/pad/face/extractor/VideoDataLoader.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jun 1 13:55:02 2017
+
+@author: Olegs Nikisins
+"""
+
+#==============================================================================
+# Import what is needed here:
+
+import os
+
+import bob.bio.video
+
+#==============================================================================
+# Main body of the class
+
+class VideoDataLoader(object):
+ """
+ This class is designed to load video data given name of the file.
+ The class is called by corresponding extractors in the experiments using
+ emty(!) preprocessor. In this scenario the video data is loaded directly
+ from the database, avoiding duplicate storage of non-processed data in the
+ experimental directory.
+
+ NOTE:
+ To use this class in PAD experiments the command line argument
+ ``--preprocessed-directory`` must point to the original database directory.
+ For example:
+ --preprocessed-directory <DIRECTORY_CONTAINING_REPLAY_ATTACK_DATABASE>
+
+ At this point the class is just a collection of methods.
+ """
+
+ #==========================================================================
+ def get_complete_filename(self, filename):
+ """
+ Get a complete filename given a filename without an extension.
+
+ **Parameters:**
+
+ ``filename`` : :py:class:`str`
+ A name of the file containing the path, but no extension.
+
+ **Returns:**
+
+ ``filename_complete`` : :py:class:`str`
+ A complete filename, incliding extension.
+ """
+
+ path, filename_no_ext = os.path.split(filename)
+
+ filenames = []
+ extensions = []
+
+ for f in os.listdir(path):
+
+ filenames.append(os.path.splitext(f)[0])
+ extensions.append(os.path.splitext(f)[1])
+
+
+ idx = filenames.index(filename_no_ext) # index of the file
+
+ file_extension = extensions[idx] # get extension of the file
+
+ filename_complete = os.path.join(path, filename_no_ext + file_extension)
+
+ return filename_complete
+
+
+ #==========================================================================
+ def load_video_data(self, filename_complete):
+ """
+ Load video data given a complete filename.
+
+ **Parameters:**
+
+ ``filename_complete`` : :py:class:`str`
+ A complete filename, incliding extension.
+
+ **Returns:**
+
+ ``video_data`` : FrameContainer
+ A FrameContainer containing the loaded video data.
+ """
+
+ frame_selector = bob.bio.video.FrameSelector(selection_style = 'all') # select all frames from the video file
+
+ video_data = frame_selector(filename_complete) # video data
+
+ return video_data
+
+
+ #==========================================================================
+ def __call__(self, filename):
+ """
+ Load video data given a filename without an extension.
+
+ **Parameters:**
+
+ ``filename`` : :py:class:`str`
+ A name of the file containing the path, but no extension.
+
+ **Returns:**
+
+ ``video_data`` : FrameContainer
+ A FrameContainer containing the loaded video data.
+ """
+
+ filename_complete = self.get_complete_filename(filename)
+
+ video_data = self.load_video_data(filename_complete)
+
+ return video_data
+
+
diff --git a/bob/pad/face/extractor/VideoLBPHistogram.py b/bob/pad/face/extractor/VideoLBPHistogram.py
index a1e1f8c9..ad920ce2 100644
--- a/bob/pad/face/extractor/VideoLBPHistogram.py
+++ b/bob/pad/face/extractor/VideoLBPHistogram.py
@@ -94,7 +94,7 @@ class VideoLBPHistogram(Extractor, object):
**Parameters:**
- ``image`` : FrameContainer
+ ``frames`` : FrameContainer
Video data stored in the FrameContainer, see ``bob.bio.video.utils.FrameContainer``
for further details.
diff --git a/bob/pad/face/extractor/VideoQualityMeasure.py b/bob/pad/face/extractor/VideoQualityMeasure.py
new file mode 100644
index 00000000..b8a938ff
--- /dev/null
+++ b/bob/pad/face/extractor/VideoQualityMeasure.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+Created on Wed May 31 16:39:34 2017
+
+@author: Olegs Nikisins
+"""
+
+#==============================================================================
+# Import what is needed here:
+
+from bob.bio.base.extractor import Extractor
+
+from bob.pad.face.extractor import ImageQualityMeasure
+
+import bob.bio.video
+
+from bob.pad.face.extractor import VideoDataLoader
+
+import six
+
+#==============================================================================
+# Main body:
+
+class VideoQualityMeasure(Extractor, object):
+ """
+ This class is designed to extract Quality Measures for each frame in the
+ input color video. For further documentation and description of features,
+ see :ref:`bob.ip.qualitymeasure`.
+
+ **Parameters:**
+
+ ``galbally`` : :py:class:`bool`
+ If ``True``, galbally features will be added to the features.
+ Default: ``True``.
+
+ ``msu`` : :py:class:`bool`
+ If ``True``, MSU features will be added to the features.
+ Default: ``True``.
+
+ ``dtype`` : numpy.dtype
+ The data type of the resulting feature vector.
+ Default: ``None``.
+ """
+
+ #==========================================================================
+ def __init__(self,
+ galbally=True,
+ msu=True,
+ dtype=None,
+ **kwargs):
+
+ super(VideoQualityMeasure, self).__init__(galbally = galbally,
+ msu = msu,
+ dtype = dtype)
+
+ self.galbally = galbally
+ self.msu = msu
+ self.dtype = dtype
+
+ # extractor to process a single image/frame:
+ extractor = ImageQualityMeasure(galbally = galbally,
+ msu = msu,
+ dtype = dtype)
+
+ # a wrapper allowing to apply above extractor to the whole video:
+ self.video_extractor = bob.bio.video.extractor.Wrapper(extractor)
+
+
+ #==========================================================================
+ def __call__(self, frames):
+ """
+ Extract feature vectors containing Quality Measures for each frame
+ in the input color video sequence/container. The resulting features
+ will be saved to the FrameContainer too.
+
+ **Parameters:**
+
+ ``frames`` : FrameContainer or string.
+ Video data stored in the FrameContainer,
+ see ``bob.bio.video.utils.FrameContainer`` for further details.
+ If string, the name of the file to load the video data from is
+ defined in it. String is possible only when empty preprocessor is
+ used. In this case video data is loaded directly from the database.
+
+ **Returns:**
+
+ ``quality_measures`` : FrameContainer
+ Quality Measures for each frame stored in the FrameContainer.
+ """
+
+ if isinstance(frames, six.string_types): # if frames is a path(!)
+
+ video_loader = VideoDataLoader()
+
+ frames = video_loader(frames) # frames is now a FrameContainer
+
+# import ipdb; ipdb.set_trace()
+
+ quality_measures = self.video_extractor(frames = frames)
+
+ return quality_measures
+
+
+ #==========================================================================
+ def write_feature(self, frames, file_name):
+ """
+ Writes the given data (that has been generated using the __call__ function of this class) to file.
+ This method overwrites the write_data() method of the Extractor class.
+
+ **Parameters:**
+
+ ``frames`` :
+ Data returned by the __call__ method of the class.
+
+ ``file_name`` : :py:class:`str`
+ Name of the file.
+ """
+
+ self.video_extractor.write_feature(frames, file_name)
+
+
+ #==========================================================================
+ def read_feature(self, file_name):
+ """
+ Reads the preprocessed data from file.
+ This method overwrites the read_data() method of the Extractor class.
+
+ **Parameters:**
+
+ ``file_name`` : :py:class:`str`
+ Name of the file.
+
+ **Returns:**
+
+ ``frames`` : :py:class:`bob.bio.video.FrameContainer`
+ Frames stored in the frame container.
+ """
+
+ frames = self.video_extractor.read_feature(file_name)
+
+ return frames
+
+
diff --git a/bob/pad/face/extractor/__init__.py b/bob/pad/face/extractor/__init__.py
index 1066793c..18f4b835 100644
--- a/bob/pad/face/extractor/__init__.py
+++ b/bob/pad/face/extractor/__init__.py
@@ -1,6 +1,8 @@
from .LBPHistogram import LBPHistogram
from .VideoLBPHistogram import VideoLBPHistogram
-
+from .ImageQualityMeasure import ImageQualityMeasure
+from .VideoDataLoader import VideoDataLoader
+from .VideoQualityMeasure import VideoQualityMeasure
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
@@ -23,5 +25,8 @@ def __appropriate__(*args):
__appropriate__(
LBPHistogram,
VideoLBPHistogram,
+ ImageQualityMeasure,
+ VideoQualityMeasure,
+ VideoDataLoader,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
diff --git a/setup.py b/setup.py
index 8c0cf3dc..4efefa4d 100644
--- a/setup.py
+++ b/setup.py
@@ -103,11 +103,13 @@ setup(
'video-face-crop-preproc-64 = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64',
'video-face-crop-preproc-64-face-50 = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64_face_50',
'video-face-crop-preproc-64-face-50-local-cropper = bob.pad.face.config.preprocessor.video_face_crop:video_face_crop_preproc_64_64_face_50_local_cropper',
+ 'empty-preprocessor = bob.pad.face.config.preprocessor.filename:empty_preprocessor', # no preprocessing
],
# registered preprocessors:
'bob.pad.extractor': [
'video-lbp-histogram-extractor-n8r1-uniform = bob.pad.face.config.extractor.video_lbp_histogram:video_lbp_histogram_extractor_n8r1_uniform',
+ 'video-quality-measure-galbally-msu = bob.pad.face.config.extractor.video_quality_measure:video_quality_measure_galbally_msu',
],
# registered algorithms:
--
GitLab