Prepare for transformers

bob/bio/video/transformer/Wrapper.py

+import logging
+
+from sklearn.base import BaseEstimator, TransformerMixin
+
+from .. import utils
+
+logger = logging.getLogger(__name__)
+
+
+class Wrapper(TransformerMixin, BaseEstimator):
+    """Wrapper class to run image preprocessing algorithms on video data.
+
+    This class provides functionality to read original video data from several databases.
+    So far, the video content from :ref:`bob.db.mobio <bob.db.mobio>` and the image list content from :ref:`bob.db.youtube <bob.db.youtube>` are supported.
+
+    Furthermore, frames are extracted from these video data, and a ``preprocessor`` algorithm is applied on all selected frames.
+    The preprocessor can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.preprocessors`, or an instance of a preprocessing class.
+    Since most of the databases do not provide annotations for all frames of the videos, commonly the preprocessor needs to apply face detection.
+
+    The ``frame_selector`` can be chosen to select some frames from the video.
+    By default, a few frames spread over the whole video sequence are selected.
+
+    The ``quality_function`` is used to assess the quality of the frame.
+    If no ``quality_function`` is given, the quality is based on the face detector, or simply left as ``None``.
+    So far, the quality of the frames are not used, but it is foreseen to select frames based on quality.
+
+    **Parameters:**
+
+    preprocessor : str or :py:class:`bob.bio.base.preprocessor.Preprocessor` instance
+      The preprocessor to be used to preprocess the frames.
+
+    frame_selector : :py:class:`bob.bio.video.FrameSelector`
+      A frame selector class to define, which frames of the video to use.
+
+    quality_function : function or ``None``
+      A function assessing the quality of the preprocessed image.
+      If ``None``, no quality assessment is performed.
+      If the preprocessor contains a ``quality`` attribute, this is taken instead.
+
+    compressed_io : bool
+      Use compression to write the resulting preprocessed HDF5 files.
+      This is experimental and might cause trouble.
+      Use this flag with care.
+
+    read_original_data: callable or ``None``
+       Function that loads the raw data.
+       If not explicitly defined the raw data will be loaded by :py:meth:`bob.bio.video.database.VideoBioFile.load`
+       using the specified ``frame_selector``
+
+    """
+
+    def __init__(
+        self,
+        transformer,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.transformer = transformer
+
+    def transform(self, videos, **kwargs):
+        transformed_videos = []
+        for i, video in enumerate(videos):
+
+            if not hasattr(video, "indices"):
+                raise ValueError(
+                    f"The input video: {video}\n does not have indices.\n "
+                    f"Processing failed in {self}"
+                )
+
+            kw = {}
+            if kwargs:
+                kw = {k: v[i] for k, v in kwargs.items()}
+            if "annotations" in kw:
+                kw["annotations"] = [
+                    kw["annotations"].get(index, kw["annotations"].get(str(index)))
+                    for index in video.indices
+                ]
+
+            data = self.transformer.transform(video, **kw)
+
+            dl, vl = len(data), len(video)
+            if dl != vl:
+                raise NotImplementedError(
+                    f"Length of transformed data ({dl}) using {self.transformer}"
+                    f" is different from the length of input video: {vl}"
+                )
+
+            # handle None's
+            indices = [idx for d, idx in zip(data, video.indices) if d is not None]
+            data = [d for d in data if d is not None]
+
+            data = utils.VideoLikeContainer(data, indices)
+            transformed_videos.append(data)
+        return transformed_videos
+
+    def _more_tags(self):
+        return {"stateless": True, "requires_fit": False}
+
+    def fit(self, X, y=None, **fit_params):
+        """Does nothing"""
+        return self

bob/bio/video/transformer/__init__.py

+from .Wrapper import Wrapper
+
+# gets sphinx autodoc done right - don't remove it
+def __appropriate__(*args):
+    """Says object was actually declared here, and not in the import module.
+    Fixing sphinx warnings of not being able to find classes, when path is shortened.
+    Parameters:
+
+      *args: An iterable of objects to modify
+
+    Resolves `Sphinx referencing issues
+    <https://github.com/sphinx-doc/sphinx/issues/3048>`
+    """
+
+    for obj in args: obj.__module__ = __name__
+
+
+__appropriate__(
+    Wrapper,
+    )
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/bio/video/utils/FrameContainer.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-
 import bob.bio.base
-import numpy
+import numpy as np
+import h5py
 import logging
 
 logger = logging.getLogger(__name__)
@@ -59,9 +57,99 @@ def select_frames(count, max_number_of_frames, selection_style, step_size):
     return indices
 
 
+class VideoAsArray:
+    def __init__(
+        self,
+        path,
+        selection_style="spread",
+        max_number_of_frames=20,
+        step_size=10,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.path = path
+        reader = self.reader
+        self.dtype, shape = reader.video_type[:2]
+        self.ndim = len(shape)
+        self.selection_style = selection_style
+        indices = select_frames(
+            count=reader.number_of_frames,
+            max_number_of_frames=max_number_of_frames,
+            selection_style=selection_style,
+            step_size=step_size,
+        )
+        self.indices = indices
+        self.shape = (len(indices),) + shape[1:]
+
+    @property
+    def reader(self):
+        return bob.io.video.reader(self.path)
+
+    def __len__(self):
+        return self.shape[0]
+
+    def __getitem__(self, index):
+        if isinstance(index, int):
+            idx = self.indices[index]
+            return self.reader[idx]
+
+        if not (isinstance(index, tuple) and len(index) == self.ndim):
+            raise NotImplementedError(f"Indxing like {index} is not supported yet!")
+
+        if all(i == slice(0, 0) for i in index):
+            return np.array([], dtype=self.dtype)
+
+        if self.selection_style == "all":
+            return np.asarray(self.reader.load())[index]
+
+        idx = self.indices[index[0]]
+        video = []
+        for i, frame in enumerate(self.reader):
+            if i not in idx:
+                continue
+            video.append(frame)
+            if i == idx[-1]:
+                break
+
+        index = (slice(len(video)),) + index[1:]
+        return np.asarray(video)[index]
+
+    def __repr__(self):
+        return f"{self.reader!r} {self.dtype!r} {self.ndim!r} {self.shape!r} {self.indices!r}"
+
+
+class VideoLikeContainer:
+    def __init__(self, data, indices, **kwargs):
+        super().__init__(**kwargs)
+        self.data = data
+        self.indices = indices
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, item):
+        return self.data[item]
+
+    def __array__(self, dtype=None, *args, **kwargs):
+        return np.asarray(self.data, dtype, *args, **kwargs)
+
+    @classmethod
+    def save(cls, other, file):
+        with h5py.File(file, mode="w") as f:
+            f["data"] = other.data
+            f["indices"] = other.indices
+
+    @classmethod
+    def load(cls, file):
+        with h5py.File(file, mode="r") as f:
+            data = np.array(f["data"])
+            indices = np.array(f["indices"])
+        self = cls(data=data, indices=indices)
+        return self
+
+
 class FrameContainer:
-    """A class for reading, manipulating and saving video content.
-    """
+    """A class for reading, manipulating and saving video content."""
 
     def __init__(self, hdf5=None, load_function=bob.bio.base.load, **kwargs):
         super().__init__(**kwargs)
@@ -163,7 +251,7 @@ class FrameContainer:
         return self
 
     def save(self, hdf5, save_function=bob.bio.base.save):
-        """ Save the content to the given HDF5 File.
+        """Save the content to the given HDF5 File.
         The contained data will be written using the given save_function."""
         if not len(self):
             logger.warn("Saving empty FrameContainer '%s'", hdf5.filename)
@@ -191,7 +279,7 @@ class FrameContainer:
                 return False
             if abs(a[2] - b[2]) > 1e-8:
                 return False
-            if not numpy.allclose(a[1], b[1]):
+            if not np.allclose(a[1], b[1]):
                 return False
         return True
 

bob/bio/video/utils/FrameSelector.py

@@ -35,9 +35,21 @@ class FrameSelector:
                 "Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')"
                 % selection_style
             )
-        self.selection = selection_style
-        self.max_frames = max_number_of_frames
-        self.step = step_size
+        self.selection_style = selection_style
+        self.max_number_of_frames = max_number_of_frames
+        self.step_size = step_size
+
+    @property
+    def selection(self):
+        return self.selection_style
+
+    @property
+    def max_frames(self):
+        return self.max_number_of_frames
+
+    @property
+    def step(self):
+        return self.step_size
 
     def __call__(self, data, load_function=bob.io.base.load):
         """Selects frames and returns them in a FrameContainer.

bob/bio/video/utils/__init__.py

-from .FrameContainer import FrameContainer, load_compressed, save_compressed, select_frames
+from .FrameContainer import FrameContainer, load_compressed, save_compressed, select_frames, VideoAsArray, VideoLikeContainer
 from .FrameSelector import FrameSelector