diff --git a/bob/pad/face/database/replay_mobile.py b/bob/pad/face/database/replay_mobile.py
index aaef0f1854716b2c5d464f9473abec24dff4e5a6..b34146e76a8edf7f2a4a2fa3237bfce77eaec1a7 100644
--- a/bob/pad/face/database/replay_mobile.py
+++ b/bob/pad/face/database/replay_mobile.py
@@ -9,8 +9,42 @@ from bob.pad.face.utils import frames, number_of_frames
from bob.db.base.annotations import read_annotation_file
import numpy
from bob.extension import rc
-# documentation imports
-import bob.bio.video
+
+REPLAYMOBILE_FRAME_SHAPE = (3, 1280, 720)
+
+
+def replaymobile_annotations(lowlevelfile, original_directory):
+ # numpy array containing the face bounding box data for each video
+ # frame, returned data format described in the f.bbx() method of the
+ # low level interface
+ annots = lowlevelfile.bbx(directory=original_directory)
+
+ annotations = {} # dictionary to return
+
+ for fn, frame_annots in enumerate(annots):
+
+ topleft = (frame_annots[1], frame_annots[0])
+ bottomright = (frame_annots[1] + frame_annots[3],
+ frame_annots[0] + frame_annots[2])
+
+ annotations[str(fn)] = {
+ 'topleft': topleft,
+ 'bottomright': bottomright
+ }
+
+ return annotations
+
+
+def replaymobile_frames(lowlevelfile, original_directory):
+ vfilename = lowlevelfile.make_path(
+ directory=original_directory,
+ extension='.mov')
+ is_not_tablet = not lowlevelfile.is_tablet()
+ for frame in frames(vfilename):
+ frame = numpy.rollaxis(frame, 2, 1)
+ if is_not_tablet:
+ frame = frame[:, ::-1, :]
+ yield frame
class ReplayMobilePadFile(VideoPadFile):
@@ -80,6 +114,34 @@ class ReplayMobilePadFile(VideoPadFile):
return frame_selector(video_data_array)
+ @property
+ def annotations(self):
+ if self.annotation_directory is not None:
+ # return the external annotations
+ annotations = read_annotation_file(
+ self.make_path(self.annotation_directory,
+ self.annotation_extension),
+ self.annotation_type)
+ return annotations
+
+ # return original annotations
+ return replaymobile_annotations(self.f, self.original_directory)
+
+ @property
+ def frames(self):
+ return replaymobile_frames(self.f, self.original_directory)
+
+ @property
+ def number_of_frames(self):
+ vfilename = self.make_path(
+ directory=self.original_directory,
+ extension='.mov')
+ return number_of_frames(vfilename)
+
+ @property
+ def frame_shape(self):
+ return REPLAYMOBILE_FRAME_SHAPE
+
class ReplayMobilePadDatabase(PadDatabase):
"""
@@ -202,6 +264,12 @@ class ReplayMobilePadDatabase(PadDatabase):
files = [ReplayMobilePadFile(f) for f in files]
+ for f in files:
+ f.original_directory = self.original_directory
+ f.annotation_directory = self.annotation_directory
+ f.annotation_extension = self.annotation_extension
+ f.annotation_type = self.annotation_type
+
return files
def annotations(self, f):
@@ -227,38 +295,11 @@ class ReplayMobilePadDatabase(PadDatabase):
A dictionary containing the annotations for each frame in the
video. Dictionary structure:
``annotations = {'1': frame1_dict, '2': frame1_dict, ...}``. Where
- ``frameN_dict = {'topleft': (row, col), 'bottomright': (row, col)}``
+ ``frameN_dict = {'topleft': (row, col),'bottomright': (row, col)}``
is the dictionary defining the coordinates of the face bounding box
in frame N.
"""
-
- if self.annotation_directory is not None:
- # return the external annotations
- annotations = read_annotation_file(
- f.make_path(self.annotation_directory,
- self.annotation_extension),
- self.annotation_type)
- return annotations
-
- # numpy array containing the face bounding box data for each video
- # frame, returned data format described in the f.bbx() method of the
- # low level interface
- annots = f.f.bbx(directory=self.original_directory)
-
- annotations = {} # dictionary to return
-
- for fn, frame_annots in enumerate(annots):
-
- topleft = (frame_annots[1], frame_annots[0])
- bottomright = (frame_annots[1] + frame_annots[3],
- frame_annots[0] + frame_annots[2])
-
- annotations[str(fn)] = {
- 'topleft': topleft,
- 'bottomright': bottomright
- }
-
- return annotations
+ return f.annotations
def frames(self, padfile):
"""Yields the frames of the padfile one by one.
@@ -273,15 +314,7 @@ class ReplayMobilePadDatabase(PadDatabase):
:any:`numpy.array`
A frame of the video. The size is (3, 1280, 720).
"""
- vfilename = padfile.make_path(
- directory=self.original_directory,
- extension=self.original_extension)
- is_not_tablet = not padfile.f.is_tablet()
- for frame in frames(vfilename):
- frame = numpy.rollaxis(frame, 2, 1)
- if is_not_tablet:
- frame = frame[:, ::-1, :]
- yield frame
+ return padfile.frames
def number_of_frames(self, padfile):
"""Returns the number of frames in a video file.
@@ -296,10 +329,7 @@ class ReplayMobilePadDatabase(PadDatabase):
int
The number of frames.
"""
- vfilename = padfile.make_path(
- directory=self.original_directory,
- extension=self.original_extension)
- return number_of_frames(vfilename)
+ return padfile.number_of_frames
@property
def frame_shape(self):
@@ -310,4 +340,4 @@ class ReplayMobilePadDatabase(PadDatabase):
(int, int, int)
The (#Channels, Height, Width) which is (3, 1280, 720).
"""
- return (3, 1280, 720)
+ return REPLAYMOBILE_FRAME_SHAPE
diff --git a/bob/pad/face/extractor/FrameDiffFeatures.py b/bob/pad/face/extractor/FrameDiffFeatures.py
index f7b2a941f20dde2985ddbfba498d117ab8a95d98..7180b73ad46e2d1340e894393297a535c36f74b3 100644
--- a/bob/pad/face/extractor/FrameDiffFeatures.py
+++ b/bob/pad/face/extractor/FrameDiffFeatures.py
@@ -246,7 +246,9 @@ class FrameDiffFeatures(Extractor):
d_bg = self.cluster_5quantities(data[:, 1], window_size, overlap)
- features = np.hstack((d_face, d_bg))
+ min_len = min(len(d_face), len(d_bg))
+
+ features = np.hstack((d_face[:min_len], d_bg[:min_len]))
frames = self.convert_arr_to_frame_cont(features)
diff --git a/bob/pad/face/preprocessor/FrameDifference.py b/bob/pad/face/preprocessor/FrameDifference.py
index 62c4ae9b1133ac20586f69c3469b02b7d4529de1..8519c69d53d2352d7ea959a949ae263f31a33188 100644
--- a/bob/pad/face/preprocessor/FrameDifference.py
+++ b/bob/pad/face/preprocessor/FrameDifference.py
@@ -225,6 +225,9 @@ class FrameDifference(Preprocessor):
# annotations for particular frame
frame_annotations = annotations[str(idx)]
+ if not frame_annotations:
+ continue
+
# Estimate bottomright and topleft if they are not available:
if 'topleft' not in frame_annotations:
bbx = bob.ip.facedetect.bounding_box_from_annotation(
diff --git a/bob/pad/face/test/dummy/database.py b/bob/pad/face/test/dummy/database.py
index dfb5c7b3bb193397ac8f3165bcaddbc980fad17b..d0e8d04ce7a13a54060f44905d0cf6a9ef104caf 100644
--- a/bob/pad/face/test/dummy/database.py
+++ b/bob/pad/face/test/dummy/database.py
@@ -4,7 +4,8 @@ import bob.io.base
import os
from bob.pad.face.database import VideoPadFile
from bob.pad.base.database import PadDatabase
-from bob.db.base.utils import check_parameters_for_validity, convert_names_to_lowlevel
+from bob.db.base.utils import (
+ check_parameters_for_validity, convert_names_to_lowlevel)
class DummyPadFile(VideoPadFile):
@@ -14,6 +15,27 @@ class DummyPadFile(VideoPadFile):
fc.add(os.path.basename(file_name), bob.io.base.load(file_name))
return fc
+ @property
+ def frames(self):
+ fc = self.load(self.original_directory)
+ for _, frame, _ in fc:
+ yield frame
+
+ @property
+ def number_of_frames(self):
+ fc = self.load(self.original_directory)
+ return len(fc)
+
+ @property
+ def frame_shape(self):
+ return (112, 92)
+
+ @property
+ def annotations(self):
+ if self.none_annotations:
+ return None
+ return {'0': {'topleft': (0, 0), 'bottomright': self.frame_shape}}
+
class DummyDatabase(PadDatabase):
@@ -33,9 +55,12 @@ class DummyDatabase(PadDatabase):
self.high_level_names = ('train', 'dev')
def _make_bio(self, files):
- return [DummyPadFile(client_id=f.client_id, path=f.path, file_id=f.id,
- attack_type=None)
- for f in files]
+ files = [DummyPadFile(client_id=f.client_id, path=f.path, file_id=f.id,
+ attack_type=None)
+ for f in files]
+ for f in files:
+ f.original_directory = self.original_directory
+ return files
def objects(self, groups=None, protocol=None, purposes=None,
model_ids=None, **kwargs):
@@ -59,13 +84,10 @@ class DummyDatabase(PadDatabase):
return None
def frames(self, padfile):
- fc = padfile.load(self.original_directory)
- for _, frame, _ in fc:
- yield frame
+ return padfile.frames
def number_of_frames(self, padfile):
- fc = padfile.load(self.original_directory)
- return len(fc)
+ return padfile.number_of_frames
@property
def frame_shape(self):
diff --git a/bob/pad/face/test/test_utils.py b/bob/pad/face/test/test_utils.py
index ade5e9a64803b95af736b789a9c4de2702fbabf4..e9b87030d418628159393ff6ff7238ed9fc059e8 100644
--- a/bob/pad/face/test/test_utils.py
+++ b/bob/pad/face/test/test_utils.py
@@ -1,6 +1,5 @@
from bob.pad.face.test.dummy.database import DummyDatabase as Database
-from bob.pad.face.utils import yield_frames, yield_faces, scale_face, blocks
-from types import MethodType
+from bob.pad.face.utils import yield_faces, scale_face, blocks
from nose.tools import raises
import numpy
@@ -13,33 +12,28 @@ def dummy_cropper(frame, annotations=None):
return frame
-def _annotations(self, padfile):
- return {'0': {'topleft': (0, 0), 'bottomright': self.frame_shape}}
-
-
def test_yield_frames():
database = Database()
- assert database.number_of_frames(padfile) == 1
- for frame in yield_frames(database, padfile):
+ nframes = database.number_of_frames(padfile)
+ assert nframes == 1, nframes
+ for frame in padfile.frames:
assert frame.ndim == 2
assert frame.shape == database.frame_shape
@raises(ValueError)
def test_yield_faces_1():
- database = Database()
- for face in yield_faces(database, padfile, dummy_cropper):
+ padfile.none_annotations = True
+ for face in yield_faces(padfile, dummy_cropper):
pass
def test_yield_faces_2():
- database = Database()
- database.annotations = MethodType(
- _annotations, database)
- assert len(list(yield_faces(database, padfile, dummy_cropper)))
- for face in yield_faces(database, padfile, dummy_cropper):
+ padfile.none_annotations = False
+ assert len(list(yield_faces(padfile, dummy_cropper)))
+ for face in yield_faces(padfile, dummy_cropper):
assert face.ndim == 2
- assert face.shape == database.frame_shape
+ assert face.shape == padfile.frame_shape
def test_scale_face():
diff --git a/bob/pad/face/utils/__init__.py b/bob/pad/face/utils/__init__.py
index 4274408459b2d3c87d8eb51a54c47817510b813a..5ef8eb5e5a50c922c1c12a7d955a00fa24e6c331 100644
--- a/bob/pad/face/utils/__init__.py
+++ b/bob/pad/face/utils/__init__.py
@@ -1,7 +1,7 @@
from .load_utils import (
- frames, number_of_frames, yield_frames, yield_faces, scale_face, blocks,
+ frames, number_of_frames, yield_faces, scale_face, blocks,
bbx_cropper, min_face_size_normalizer, color_augmentation,
- the_giant_video_loader)
+ blocks_generator, the_giant_video_loader)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
diff --git a/bob/pad/face/utils/load_utils.py b/bob/pad/face/utils/load_utils.py
index bad3b93b81298c69cf71a3d56cee1c1a56546fde..e37ece1f13c8d06d0abbddb4a89bcacfdb700358 100644
--- a/bob/pad/face/utils/load_utils.py
+++ b/bob/pad/face/utils/load_utils.py
@@ -1,12 +1,13 @@
from bob.bio.face.annotator import min_face_size_validator
from bob.bio.video.annotator import normalize_annotations
from bob.io.video import reader
-from bob.ip.base import scale, block, block_output_shape
+from bob.ip.base import scale, block, block_output_shape, block_generator
from bob.ip.color import rgb_to_yuv, rgb_to_hsv
from bob.ip.facedetect import bounding_box_from_annotation
from collections import OrderedDict
from functools import partial
import numpy
+import random
def frames(path):
@@ -43,25 +44,6 @@ def number_of_frames(path):
return video.number_of_frames
-def yield_frames(paddb, padfile):
- """Loads the frames of a video PAD database.
-
- Parameters
- ----------
- paddb : :any:`bob.pad.base.database.PadDatabase`
- The video PAD database. The database needs to have implemented the
- `.frames()` method.
- padfile : :any:`bob.pad.face.database.VideoPadFile`
- The PAD file.
-
- Yields
- ------
- :any:`numpy.array`
- Frames of the PAD file one by one.
- """
- return paddb.frames(padfile)
-
-
def bbx_cropper(frame, annotations):
bbx = bounding_box_from_annotation(**annotations)
return frame[..., bbx.top:bbx.bottom, bbx.left:bbx.right]
@@ -73,15 +55,12 @@ def min_face_size_normalizer(annotations, max_age=15, **kwargs):
max_age=max_age)
-def yield_faces(database, padfile, cropper, normalizer=None):
+def yield_faces(padfile, cropper, normalizer=None):
"""Yields face images of a padfile. It uses the annotations from the
database. The annotations are further normalized.
Parameters
----------
- database : :any:`bob.pad.base.database.PadDatabase`
- A face PAD database. This database needs to have implemented the
- `frames` method.
padfile : :any:`bob.pad.base.database.PadFile`
The padfile to return the faces.
cropper : callable
@@ -101,10 +80,10 @@ def yield_faces(database, padfile, cropper, normalizer=None):
ValueError
If the database returns None for annotations.
"""
- frames_gen = database.frames(padfile)
+ frames_gen = padfile.frames
# read annotation
- annotations = database.annotations(padfile)
+ annotations = padfile.annotations
if annotations is None:
raise ValueError("No annotations were returned.")
@@ -192,6 +171,41 @@ def blocks(data, block_size, block_overlap=(0, 0)):
return output
+def blocks_generator(data, block_size, block_overlap=(0, 0)):
+ """Yields patches of an image
+
+ Parameters
+ ----------
+ data : :any:`numpy.array`
+ The image in gray-scale, color, or color video format.
+ block_size : (int, int)
+ The size of patches
+ block_overlap : (:obj:`int`, :obj:`int`), optional
+ The size of overlap of patches
+
+ Yields
+ ------
+ :any:`numpy.array`
+ The patches.
+
+ Raises
+ ------
+ ValueError
+ If data dimension is not between 2 and 4 (inclusive).
+ """
+ data = numpy.asarray(data)
+ if 1 < data.ndim < 4:
+ for patch in block_generator(data, block_size, block_overlap):
+ yield patch
+ # if a color video:
+ elif data.ndim == 4:
+ for frame in data:
+ for patch in block_generator(frame, block_size, block_overlap):
+ yield patch
+ else:
+ raise ValueError("Unknown data dimension {}".format(data.ndim))
+
+
def color_augmentation(image, channels=('rgb',)):
"""Converts an RGB image to different color channels.
@@ -232,12 +246,55 @@ def the_giant_video_loader(paddb, padfile,
normalizer=None, patches=False,
block_size=(96, 96), block_overlap=(0, 0),
random_patches_per_frame=None, augment=None,
- multiple_bonafide_patches=1):
+ multiple_bonafide_patches=1, keep_pa_samples=None):
+ """Loads a video pad file frame by frame and optionally applies
+ transformations.
+
+ Parameters
+ ----------
+ paddb
+ Ignored.
+ padfile
+ The pad file
+ region : str
+ Either `whole` or `crop`. If whole, it will return the whole frame.
+ Otherwise, you need to provide a cropper and a normalizer.
+ scaling_factor : float
+ If given, will scale images to this factor.
+ cropper
+ The cropper to use
+ normalizer
+ The normalizer to use
+ patches : bool
+ If true, will extract patches from images.
+ block_size : tuple
+ Size of the patches
+ block_overlap : tuple
+ Size of overlap of the patches
+ random_patches_per_frame : int
+ If not None, will only take this much patches per frame
+ augment
+ If given, frames will be transformed using this function.
+ multiple_bonafide_patches : int
+ Will use more random patches for bonafide samples
+ keep_pa_samples : float
+ If given, will drop some PA samples.
+
+ Returns
+ -------
+ object
+ A generator that yields the samples.
+
+ Raises
+ ------
+ ValueError
+ If region is not whole or crop.
+ """
if region == 'whole':
- generator = yield_frames(paddb, padfile)
+ generator = padfile.frames
elif region == 'crop':
generator = yield_faces(
- paddb, padfile, cropper=cropper, normalizer=normalizer)
+ padfile, cropper=cropper, normalizer=normalizer)
else:
raise ValueError("Invalid region value: `{}'".format(region))
@@ -248,7 +305,8 @@ def the_giant_video_loader(paddb, padfile,
if random_patches_per_frame is None:
generator = (
patch for frame in generator
- for patch in blocks(frame, block_size, block_overlap))
+ for patch in blocks_generator(
+ frame, block_size, block_overlap))
else:
if padfile.attack_type is None:
random_patches_per_frame *= multiple_bonafide_patches
@@ -261,4 +319,8 @@ def the_giant_video_loader(paddb, padfile,
if augment is not None:
generator = (augment(frame) for frame in generator)
+ if keep_pa_samples is not None and padfile.attack_type is not None:
+ generator = (frame for frame in generator
+ if random.random() < keep_pa_samples)
+
return generator
diff --git a/doc/api.rst b/doc/api.rst
index fe623bb207a004423bff797e55c1e5653bc886f3..04e403c56f78a833559fa1b07102fc28f03e0f65 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -73,6 +73,7 @@ Utilities
.. autosummary::
bob.pad.face.utils.bbx_cropper
bob.pad.face.utils.blocks
+ bob.pad.face.utils.blocks_generator
bob.pad.face.utils.color_augmentation
bob.pad.face.utils.frames
bob.pad.face.utils.min_face_size_normalizer
@@ -80,6 +81,6 @@ Utilities
bob.pad.face.utils.scale_face
bob.pad.face.utils.the_giant_video_loader
bob.pad.face.utils.yield_faces
- bob.pad.face.utils.yield_frames
+
.. automodule:: bob.pad.face.utils