Skip to content
GitLab
Commit 7642808d authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Prepare for transformers

parent 35f6426d
Pipeline #44391 failed with stage
in 3 minutes and 58 seconds
Hide whitespace changes
Inline Side-by-side
bob/bio/video/extractor/Wrapper.py
View file @ 7642808d
......@@ -3,171 +3,179 @@
import bob.bio.base
import bob.io.base
import os
import six
from .. import utils
class Wrapper (bob.bio.base.extractor.Extractor):
"""Wrapper class to run feature extraction algorithms on frame containers.
Features are extracted for all frames in the frame container using the provided ``extractor``.
The ``extractor`` can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.extractors`, or an instance of an extractor class.
The ``frame_selector`` can be chosen to select some frames from the frame container.
By default, all frames from the previous preprocessing step are kept, but fewer frames might be selected in this stage.
**Parameters:**
extractor : str or :py:class:`bob.bio.base.extractor.Extractor` instance
The extractor to be used to extract features from the frames.
frame_selector : :py:class:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the preprocessed frame container to use.
compressed_io : bool
Use compression to write the resulting features to HDF5 files.
This is experimental and might cause trouble.
Use this flag with care.
"""
def __init__(self,
extractor,
frame_selector = utils.FrameSelector(selection_style='all'),
compressed_io = False
):
# load extractor configuration
if isinstance(extractor, six.string_types):
self.extractor = bob.bio.base.load_resource(extractor, "extractor")
elif isinstance(extractor, bob.bio.base.extractor.Extractor):
self.extractor = extractor
else:
raise ValueError("The given extractor could not be interpreted")
self.frame_selector = frame_selector
self.compressed_io = compressed_io
# register extractor's details
bob.bio.base.extractor.Extractor.__init__(
self,
requires_training=self.extractor.requires_training,
split_training_data_by_client=self.extractor.split_training_data_by_client,
extractor=extractor,
frame_selector=frame_selector,
compressed_io=compressed_io
)
def _check_feature(self, frames):
"""Checks if the given feature is in the desired format."""
assert isinstance(frames, utils.FrameContainer)
def __call__(self, frames):
"""__call__(frames) -> features
Extracts the frames from the video and returns a frame container.
This function is used to extract features using the desired ``extractor`` for all frames that are selected by the ``frame_selector`` specified in the constructor of this class.
**Parameters:**
frames : :py:class:`bob.bio.video.FrameContainer`
The frame container containing preprocessed image frames.
**Returns:**
class Wrapper(bob.bio.base.extractor.Extractor):
"""Wrapper class to run feature extraction algorithms on frame containers.
features : :py:class:`bob.bio.video.FrameContainer`
A frame container containing extracted features.
"""
self._check_feature(frames)
# go through the frames and extract the features
fc = utils.FrameContainer()
for index, frame, quality in self.frame_selector(frames):
# extract features
extracted = self.extractor(frame)
if extracted is not None:
# add features to new frame container
fc.add(index, extracted, quality)
if not len(fc):
return None
return fc
def read_feature(self, filename):
"""read_feature(filename) -> frames
Features are extracted for all frames in the frame container using the provided ``extractor``.
The ``extractor`` can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.extractors`, or an instance of an extractor class.
Reads the extracted data from file and returns them in a frame container.
The extractors ``read_feature`` function is used to read the data for each frame.
The ``frame_selector`` can be chosen to select some frames from the frame container.
By default, all frames from the previous preprocessing step are kept, but fewer frames might be selected in this stage.
**Parameters:**
filename : str
The name of the extracted data file.
extractor : str or :py:class:`bob.bio.base.extractor.Extractor` instance
The extractor to be used to extract features from the frames.
**Returns:**
frame_selector : :py:class:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the preprocessed frame container to use.
frames : :py:class:`bob.bio.video.FrameContainer`
The read frames, stored in a frame container.
compressed_io : bool
Use compression to write the resulting features to HDF5 files.
This is experimental and might cause trouble.
Use this flag with care.
"""
if self.compressed_io:
return utils.load_compressed(filename, self.extractor.read_feature)
else:
return utils.FrameContainer(bob.io.base.HDF5File(filename), self.extractor.read_feature)
def __init__(
self,
extractor,
frame_selector=utils.FrameSelector(selection_style="all"),
compressed_io=False,
):
# load extractor configuration
if isinstance(extractor, str):
extractor = bob.bio.base.load_resource(extractor, "extractor")
self.extractor = extractor
def write_feature(self, frames, filename):
"""Writes the extracted features to file.
The extractors ``write_features`` function is used to write the features for each frame.
**Parameters:**
frames : :py:class:`bob.bio.video.FrameContainer`
The extracted features for the selected frames, as returned by the `__call__` function.
filename : str
The file name to write the extracted feature into.
"""
self._check_feature(frames)
if self.compressed_io:
return utils.save_compressed(frames, filename, self.extractor.write_feature)
else:
frames.save(bob.io.base.HDF5File(filename, 'w'), self.extractor.write_feature)
def train(self, training_frames, extractor_file):
"""Trains the feature extractor with the preprocessed data of the given frames.
.. note::
This function is not called, when the given ``extractor`` does not require training.
This function will train the feature extractor using all data from the selected frames of the training data.
The training_frames must be aligned by client if the given ``extractor`` requires that.
**Parameters:**
training_frames : [:py:class:`bob.bio.video.FrameContainer`] or [[:py:class:`bob.bio.video.FrameContainer`]]
The set of training frames, which will be used to train the ``extractor``.
extractor_file : str
The name of the extractor that should be written.
"""
if self.split_training_data_by_client:
[self._check_feature(frames) for client_frames in training_frames for frames in client_frames]
features = [[frame[1] for frames in client_frames for frame in self.frame_selector(frames)] for client_frames in training_frames]
else:
[self._check_feature(frames) for frames in training_frames]
features = [frame[1] for frames in training_frames for frame in self.frame_selector(frames)]
self.extractor.train(features, extractor_file)
def load(self, extractor_file):
"""Loads the trained extractor from file.
This function calls the wrapped classes ``load`` function.
extractor_file : str
The name of the extractor that should be loaded.
"""
self.extractor.load(extractor_file)
self.frame_selector = frame_selector
self.compressed_io = compressed_io
# register extractor's details
super().__init__(
self,
requires_training=self.extractor.requires_training,
split_training_data_by_client=self.extractor.split_training_data_by_client,
)
def _check_feature(self, frames):
"""Checks if the given feature is in the desired format."""
assert isinstance(frames, utils.FrameContainer)
def __call__(self, frames):
"""__call__(frames) -> features
Extracts the frames from the video and returns a frame container.
This function is used to extract features using the desired ``extractor`` for all frames that are selected by the ``frame_selector`` specified in the constructor of this class.
**Parameters:**
frames : :py:class:`bob.bio.video.FrameContainer`
The frame container containing preprocessed image frames.
**Returns:**
features : :py:class:`bob.bio.video.FrameContainer`
A frame container containing extracted features.
"""
self._check_feature(frames)
# go through the frames and extract the features
fc = utils.FrameContainer()
for index, frame, quality in self.frame_selector(frames):
# extract features
extracted = self.extractor(frame)
if extracted is not None:
# add features to new frame container
fc.add(index, extracted, quality)
if not len(fc):
return None
return fc
def read_feature(self, filename):
"""read_feature(filename) -> frames
Reads the extracted data from file and returns them in a frame container.
The extractors ``read_feature`` function is used to read the data for each frame.
**Parameters:**
filename : str
The name of the extracted data file.
**Returns:**
frames : :py:class:`bob.bio.video.FrameContainer`
The read frames, stored in a frame container.
"""
if self.compressed_io:
return utils.load_compressed(filename, self.extractor.read_feature)
else:
return utils.FrameContainer(
bob.io.base.HDF5File(filename), self.extractor.read_feature
)
def write_feature(self, frames, filename):
"""Writes the extracted features to file.
The extractors ``write_features`` function is used to write the features for each frame.
**Parameters:**
frames : :py:class:`bob.bio.video.FrameContainer`
The extracted features for the selected frames, as returned by the `__call__` function.
filename : str
The file name to write the extracted feature into.
"""
self._check_feature(frames)
if self.compressed_io:
return utils.save_compressed(frames, filename, self.extractor.write_feature)
else:
frames.save(
bob.io.base.HDF5File(filename, "w"), self.extractor.write_feature
)
def train(self, training_frames, extractor_file):
"""Trains the feature extractor with the preprocessed data of the given frames.
.. note::
This function is not called, when the given ``extractor`` does not require training.
This function will train the feature extractor using all data from the selected frames of the training data.
The training_frames must be aligned by client if the given ``extractor`` requires that.
**Parameters:**
training_frames : [:py:class:`bob.bio.video.FrameContainer`] or [[:py:class:`bob.bio.video.FrameContainer`]]
The set of training frames, which will be used to train the ``extractor``.
extractor_file : str
The name of the extractor that should be written.
"""
if self.split_training_data_by_client:
[
self._check_feature(frames)
for client_frames in training_frames
for frames in client_frames
]
features = [
[
frame[1]
for frames in client_frames
for frame in self.frame_selector(frames)
]
for client_frames in training_frames
]
else:
[self._check_feature(frames) for frames in training_frames]
features = [
frame[1]
for frames in training_frames
for frame in self.frame_selector(frames)
]
self.extractor.train(features, extractor_file)
def load(self, extractor_file):
"""Loads the trained extractor from file.
This function calls the wrapped classes ``load`` function.
extractor_file : str
The name of the extractor that should be loaded.
"""
self.extractor.load(extractor_file)
bob/bio/video/transformer/Wrapper.py 0 → 100644
View file @ 7642808d
import logging
from sklearn.base import BaseEstimator, TransformerMixin
from .. import utils
logger = logging.getLogger(__name__)
class Wrapper(TransformerMixin, BaseEstimator):
"""Wrapper class to run image preprocessing algorithms on video data.
This class provides functionality to read original video data from several databases.
So far, the video content from :ref:`bob.db.mobio <bob.db.mobio>` and the image list content from :ref:`bob.db.youtube <bob.db.youtube>` are supported.
Furthermore, frames are extracted from these video data, and a ``preprocessor`` algorithm is applied on all selected frames.
The preprocessor can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.preprocessors`, or an instance of a preprocessing class.
Since most of the databases do not provide annotations for all frames of the videos, commonly the preprocessor needs to apply face detection.
The ``frame_selector`` can be chosen to select some frames from the video.
By default, a few frames spread over the whole video sequence are selected.
The ``quality_function`` is used to assess the quality of the frame.
If no ``quality_function`` is given, the quality is based on the face detector, or simply left as ``None``.
So far, the quality of the frames are not used, but it is foreseen to select frames based on quality.
**Parameters:**
preprocessor : str or :py:class:`bob.bio.base.preprocessor.Preprocessor` instance
The preprocessor to be used to preprocess the frames.
frame_selector : :py:class:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the video to use.
quality_function : function or ``None``
A function assessing the quality of the preprocessed image.
If ``None``, no quality assessment is performed.
If the preprocessor contains a ``quality`` attribute, this is taken instead.
compressed_io : bool
Use compression to write the resulting preprocessed HDF5 files.
This is experimental and might cause trouble.
Use this flag with care.
read_original_data: callable or ``None``
Function that loads the raw data.
If not explicitly defined the raw data will be loaded by :py:meth:`bob.bio.video.database.VideoBioFile.load`
using the specified ``frame_selector``
"""
def __init__(
self,
transformer,
**kwargs,
):
super().__init__(**kwargs)
self.transformer = transformer
def transform(self, videos, **kwargs):
transformed_videos = []
for i, video in enumerate(videos):
if not hasattr(video, "indices"):
raise ValueError(
f"The input video: {video}\n does not have indices.\n "
f"Processing failed in {self}"
)
kw = {}
if kwargs:
kw = {k: v[i] for k, v in kwargs.items()}
if "annotations" in kw:
kw["annotations"] = [
kw["annotations"].get(index, kw["annotations"].get(str(index)))
for index in video.indices
]
data = self.transformer.transform(video, **kw)
dl, vl = len(data), len(video)
if dl != vl:
raise NotImplementedError(
f"Length of transformed data ({dl}) using {self.transformer}"
f" is different from the length of input video: {vl}"
)
# handle None's
indices = [idx for d, idx in zip(data, video.indices) if d is not None]
data = [d for d in data if d is not None]
data = utils.VideoLikeContainer(data, indices)
transformed_videos.append(data)
return transformed_videos
def _more_tags(self):
return {"stateless": True, "requires_fit": False}
def fit(self, X, y=None, **fit_params):
"""Does nothing"""
return self
bob/bio/video/transformer/__init__.py 0 → 100644
View file @ 7642808d
from .Wrapper import Wrapper
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args: obj.__module__ = __name__
__appropriate__(
Wrapper,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
bob/bio/video/utils/FrameContainer.py
View file @ 7642808d
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import bob.bio.base
import numpy
import numpy as np
import h5py
import logging
logger = logging.getLogger(__name__)
......@@ -59,9 +57,99 @@ def select_frames(count, max_number_of_frames, selection_style, step_size):
return indices
class VideoAsArray:
def __init__(
self,
path,
selection_style="spread",
max_number_of_frames=20,
step_size=10,
**kwargs,
):
super().__init__(**kwargs)
self.path = path
reader = self.reader
self.dtype, shape = reader.video_type[:2]
self.ndim = len(shape)
self.selection_style = selection_style
indices = select_frames(
count=reader.number_of_frames,
max_number_of_frames=max_number_of_frames,
selection_style=selection_style,
step_size=step_size,
)
self.indices = indices
self.shape = (len(indices),) + shape[1:]
@property
def reader(self):
return bob.io.video.reader(self.path)
def __len__(self):
return self.shape[0]
def __getitem__(self, index):
if isinstance(index, int):
idx = self.indices[index]
return self.reader[idx]
if not (isinstance(index, tuple) and len(index) == self.ndim):
raise NotImplementedError(f"Indxing like {index} is not supported yet!")
if all(i == slice(0, 0) for i in index):
return np.array([], dtype=self.dtype)
if self.selection_style == "all":
return np.asarray(self.reader.load())[index]
idx = self.indices[index[0]]
video = []
for i, frame in enumerate(self.reader):
if i not in idx:
continue
video.append(frame)
if i == idx[-1]:
break
index = (slice(len(video)),) + index[1:]
return np.asarray(video)[index]
def __repr__(self):
return f"{self.reader!r} {self.dtype!r} {self.ndim!r} {self.shape!r} {self.indices!r}"
class VideoLikeContainer:
def __init__(self, data, indices, **kwargs):
super().__init__(**kwargs)
self.data = data
self.indices = indices
def __len__(self):
return len(self.data)
def __getitem__(self, item):
return self.data[item]
def __array__(self, dtype=None, *args, **kwargs):
return np.asarray(self.data, dtype, *args, **kwargs)
@classmethod
def save(cls, other, file):
with h5py.File(file, mode="w") as f:
f["data"] = other.data
f["indices"] = other.indices
@classmethod
def load(cls, file):
with h5py.File(file, mode="r") as f:
data = np.array(f["data"])
indices = np.array(f["indices"])
self = cls(data=data, indices=indices)
return self
class FrameContainer:
"""A class for reading, manipulating and saving video content.
"""
"""A class for reading, manipulating and saving video content."""
def __init__(self, hdf5=None, load_function=bob.bio.base.load, **kwargs):
super().__init__(**kwargs)
......@@ -163,7 +251,7 @@ class FrameContainer:
return self
def save(self, hdf5, save_function=bob.bio.base.save):
""" Save the content to the given HDF5 File.
"""Save the content to the given HDF5 File.
The contained data will be written using the given save_function."""
if not len(self):
logger.warn("Saving empty FrameContainer '%s'", hdf5.filename)
......@@ -191,7 +279,7 @@ class FrameContainer:
return False
if abs(a[2] - b[2]) > 1e-8:
return False
if not numpy.allclose(a[1], b[1]):
if not np.allclose(a[1], b[1]):
return False
return True
......
bob/bio/video/utils/FrameSelector.py
View file @ 7642808d
......@@ -35,9 +35,21 @@ class FrameSelector:
"Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')"
% selection_style
)
self.selection = selection_style
self.max_frames = max_number_of_frames
self.step = step_size
self.selection_style = selection_style
self.max_number_of_frames = max_number_of_frames
self.step_size = step_size
@property
def selection(self):
return self.selection_style
@property
def max_frames(self):
return self.max_number_of_frames
@property
def step(self):
return self.step_size