Commit 44df55d2 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

[refactoring2016] Moved the load functions to the to the high level implementations

parent a02fcfdb
Pipeline #3695 passed with stages
in 173 minutes and 55 seconds
...@@ -2,7 +2,7 @@ from .utils import * ...@@ -2,7 +2,7 @@ from .utils import *
from . import preprocessor from . import preprocessor
from . import extractor from . import extractor
from . import algorithm from . import algorithm
from . import database
from . import test from . import test
def get_config(): def get_config():
......
from .database import VideoBioFile
from .mobio import MobioBioDatabase
from .youtube import YoutubeBioDatabase
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
""" """
from bob.bio.base.database.file import BioFile from bob.bio.base.database.file import BioFile
from bob.bio.video.utils.FrameSelector import FrameSelector
class VideoBioFile(BioFile): class VideoBioFile(BioFile):
...@@ -20,4 +21,8 @@ class VideoBioFile(BioFile): ...@@ -20,4 +21,8 @@ class VideoBioFile(BioFile):
self.__f = f self.__f = f
def load(self, directory=None, extension='.avi'):
return FrameSelector()(self.make_path(directory, extension))
...@@ -12,6 +12,21 @@ ...@@ -12,6 +12,21 @@
from .database import VideoBioFile from .database import VideoBioFile
from bob.bio.base.database import ZTBioDatabase, BioFile from bob.bio.base.database import ZTBioDatabase, BioFile
from bob.bio.video.utils import FrameContainer
import os
import bob.io.base
class YoutubeBioFile(VideoBioFile):
def load(self, directory=None, extension='.jpg'):
files = os.listdir(self.make_path(directory, ''))
fc = FrameContainer()
for f in files:
if extension == os.path.splitext(f)[1]:
file_name = os.path.join(self.make_path(directory, ''), f)
fc.add(os.path.basename(file_name), bob.io.base.load(file_name))
return fc
class YoutubeBioDatabase(ZTBioDatabase): class YoutubeBioDatabase(ZTBioDatabase):
...@@ -38,12 +53,12 @@ class YoutubeBioDatabase(ZTBioDatabase): ...@@ -38,12 +53,12 @@ class YoutubeBioDatabase(ZTBioDatabase):
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs): def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
retval = self.__db.objects(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids, **kwargs) retval = self.__db.objects(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids, **kwargs)
return [VideoBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval] return [YoutubeBioFile(BioFile(client_id=f.client_id, path=f.path, file_id=f.id)) for f in retval]
def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs): def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
retval = self.__db.tobjects(groups=groups, protocol=protocol, model_ids=model_ids, **kwargs) retval = self.__db.tobjects(groups=groups, protocol=protocol, model_ids=model_ids, **kwargs)
return [VideoBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval] return [YoutubeBioFile(BioFile(client_id=f.client_id, path=f.path, file_id=f.id)) for f in retval]
def zobjects(self, groups=None, protocol=None, **kwargs): def zobjects(self, groups=None, protocol=None, **kwargs):
retval = self.__db.zobjects(groups=groups, protocol=protocol, **kwargs) retval = self.__db.zobjects(groups=groups, protocol=protocol, **kwargs)
return [VideoBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval] return [YoutubeBioFile(BioFile(client_id=f.client_id, path=f.path, file_id=f.id)) for f in retval]
...@@ -2,195 +2,169 @@ ...@@ -2,195 +2,169 @@
# vim: set fileencoding=utf-8 : # vim: set fileencoding=utf-8 :
import bob.bio.base import bob.bio.base
import numpy
import glob
import os
import bob.io.base import bob.io.base
from .. import utils from .. import utils
class Wrapper (bob.bio.base.preprocessor.Preprocessor):
"""Wrapper class to run image preprocessing algorithms on video data.
This class provides functionality to read original video data from several databases.
So far, the video content from :py:class:`bob.db.mobio` and the image list content from :py:class:`bob.db.youtube` are supported.
Furthermore, frames are extracted from these video data, and a ``preprocessor`` algorithm is applied on all selected frames.
The preprocessor can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.preprocessors`, or an instance of a preprocessing class.
Since most of the databases do not provide annotations for all frames of the videos, commonly the preprocessor needs to apply face detection.
The ``frame_selector`` can be chosen to select some frames from the video. class Wrapper(bob.bio.base.preprocessor.Preprocessor):
By default, a few frames spread over the whole video sequence are selected. """Wrapper class to run image preprocessing algorithms on video data.
The ``quality_function`` is used to assess the quality of the frame. This class provides functionality to read original video data from several databases.
If no ``quality_function`` is given, the quality is based on the face detector, or simply left as ``None``. So far, the video content from :py:class:`bob.db.mobio` and the image list content from :py:class:`bob.db.youtube` are supported.
So far, the quality of the frames are not used, but it is foreseen to select frames based on quality.
**Parameters:** Furthermore, frames are extracted from these video data, and a ``preprocessor`` algorithm is applied on all selected frames.
The preprocessor can either be provided as a registered resource, i.e., one of :ref:`bob.bio.face.preprocessors`, or an instance of a preprocessing class.
Since most of the databases do not provide annotations for all frames of the videos, commonly the preprocessor needs to apply face detection.
preprocessor : str or :py:class:`bob.bio.base.preprocessor.Preprocessor` instance The ``frame_selector`` can be chosen to select some frames from the video.
The preprocessor to be used to preprocess the frames. By default, a few frames spread over the whole video sequence are selected.
frame_selector : :py:class:`bob.bio.video.FrameSelector` The ``quality_function`` is used to assess the quality of the frame.
A frame selector class to define, which frames of the video to use. If no ``quality_function`` is given, the quality is based on the face detector, or simply left as ``None``.
So far, the quality of the frames are not used, but it is foreseen to select frames based on quality.
quality_function : function or ``None`` **Parameters:**
A function assessing the quality of the preprocessed image.
If ``None``, no quality assessment is performed.
If the preprocessor contains a ``quality`` attribute, this is taken instead.
compressed_io : bool
Use compression to write the resulting preprocessed HDF5 files.
This is experimental and might cause trouble.
Use this flag with care.
"""
def __init__(self, preprocessor : str or :py:class:`bob.bio.base.preprocessor.Preprocessor` instance
preprocessor = 'landmark-detect', The preprocessor to be used to preprocess the frames.
frame_selector = utils.FrameSelector(),
quality_function = None,
compressed_io = False
):
# load preprocessor configuration frame_selector : :py:class:`bob.bio.video.FrameSelector`
if isinstance(preprocessor, str): A frame selector class to define, which frames of the video to use.
self.preprocessor = bob.bio.base.load_resource(preprocessor, "preprocessor")
elif isinstance(preprocessor, bob.bio.base.preprocessor.Preprocessor):
self.preprocessor = preprocessor
else:
raise ValueError("The given preprocessor could not be interpreted")
bob.bio.base.preprocessor.Preprocessor.__init__( quality_function : function or ``None``
self, A function assessing the quality of the preprocessed image.
preprocessor=preprocessor, If ``None``, no quality assessment is performed.
frame_selector=frame_selector, If the preprocessor contains a ``quality`` attribute, this is taken instead.
compressed_io=compressed_io
)
self.frame_selector = frame_selector compressed_io : bool
self.quality_function = quality_function Use compression to write the resulting preprocessed HDF5 files.
self.compressed_io = compressed_io This is experimental and might cause trouble.
Use this flag with care.
"""
def _check_data(self, frames): def __init__(self,
"""Checks if the given video is in the desired format.""" preprocessor='landmark-detect',
assert isinstance(frames, utils.FrameContainer) frame_selector=utils.FrameSelector(),
quality_function=None,
compressed_io=False,
read_original_data=lambda biofile, directory, extension: biofile.load(directory, extension)
):
# load preprocessor configuration
if isinstance(preprocessor, str):
self.preprocessor = bob.bio.base.load_resource(preprocessor, "preprocessor")
elif isinstance(preprocessor, bob.bio.base.preprocessor.Preprocessor):
self.preprocessor = preprocessor
else:
raise ValueError("The given preprocessor could not be interpreted")
bob.bio.base.preprocessor.Preprocessor.__init__(
self,
preprocessor=preprocessor,
frame_selector=frame_selector,
compressed_io=compressed_io,
read_original_data=read_original_data
)
def __call__(self, frames, annotations = None): self.frame_selector = frame_selector
"""__call__(frames, annotations = None) -> preprocessed self.quality_function = quality_function
self.compressed_io = compressed_io
Preprocesses the given frames using the desired ``preprocessor``. def _check_data(self, frames):
"""Checks if the given video is in the desired format."""
assert isinstance(frames, utils.FrameContainer)
Faces are extracted for all frames in the given frame container, using the ``preprocessor`` specified in the constructor. def __call__(self, frames, annotations=None):
"""__call__(frames, annotations = None) -> preprocessed
If given, the annotations need to be in a dictionary. Preprocesses the given frames using the desired ``preprocessor``.
The key is either the frame number (for video data) or the image name (for image list data).
The value is another dictionary, building the relation between facial landmark names and their location, e.g. ``{'leye' : (le_y, le_x), 'reye' : (re_y, re_x)}``
The annotations for the according frames, if present, are passed to the preprocessor. Faces are extracted for all frames in the given frame container, using the ``preprocessor`` specified in the constructor.
Please assure that your database interface provides the annotations in the desired format.
**Parameters:** If given, the annotations need to be in a dictionary.
The key is either the frame number (for video data) or the image name (for image list data).
The value is another dictionary, building the relation between facial landmark names and their location, e.g. ``{'leye' : (le_y, le_x), 'reye' : (re_y, re_x)}``
frames : :py:class:`bob.bio.video.FrameContainer` The annotations for the according frames, if present, are passed to the preprocessor.
The pre-selected frames, as returned by :py:meth:`read_original_data`. Please assure that your database interface provides the annotations in the desired format.
annotations : dict or ``None`` **Parameters:**
The annotations for the frames, if any.
**Returns:** frames : :py:class:`bob.bio.video.FrameContainer`
The pre-selected frames, as returned by :py:meth:`read_original_data`.
preprocessed : :py:class:`bob.bio.video.FrameContainer` annotations : dict or ``None``
A frame container that contains the preprocessed frames. The annotations for the frames, if any.
"""
if not isinstance(frames, utils.FrameContainer):
frames = self.frame_selector(frames)
annots = None
fc = utils.FrameContainer()
for index, frame, _ in frames:
# if annotations are given, we take them
if annotations is not None: annots = annotations[index]
# preprocess image (by default: detect a face)
preprocessed = self.preprocessor(frame, annots)
if preprocessed is not None:
# compute the quality of the detection
if self.quality_function is not None:
quality = self.quality_function(preprocessed)
elif hasattr(self.preprocessor, 'quality'):
quality = self.preprocessor.quality
else:
quality = None
# add image to frame container
fc.add(index, preprocessed, quality)
return fc **Returns:**
preprocessed : :py:class:`bob.bio.video.FrameContainer`
A frame container that contains the preprocessed frames.
"""
if not isinstance(frames, utils.FrameContainer):
frames = self.frame_selector(frames)
def read_original_data(self, data): annots = None
"""read_original_data(data) -> frames fc = utils.FrameContainer()
Reads the original data from file and selects some frames using the desired ``frame_selector``. for index, frame, _ in frames:
# if annotations are given, we take them
if annotations is not None: annots = annotations[index]
Currently, two types of data is supported: # preprocess image (by default: detect a face)
preprocessed = self.preprocessor(frame, annots)
if preprocessed is not None:
# compute the quality of the detection
if self.quality_function is not None:
quality = self.quality_function(preprocessed)
elif hasattr(self.preprocessor, 'quality'):
quality = self.preprocessor.quality
else:
quality = None
# add image to frame container
fc.add(index, preprocessed, quality)
1. video data, which is stored in a 3D or 4D :py:class:`numpy.ndarray`, which will be read using :py:func:`bob.io.base.load` return fc
2. image lists, which is given as a list of strings of image file names. Each image will be read with :py:func:`bob.io.base.load`
**Parameters:**
data : 3D or 4D :py:class:`numpy.ndarray`, or [str] def read_data(self, filename):
The original data to read. """read_data(filename) -> frames
**Returns:** Reads the preprocessed data from file and returns them in a frame container.
The preprocessors ``read_data`` function is used to read the data for each frame.
frames : :py:class:`bob.bio.video.FrameContainer` **Parameters:**
The selected frames, stored in a frame container.
"""
return self.frame_selector(data)
filename : str
The name of the preprocessed data file.
def read_data(self, filename): **Returns:**
"""read_data(filename) -> frames
Reads the preprocessed data from file and returns them in a frame container. frames : :py:class:`bob.bio.video.FrameContainer`
The preprocessors ``read_data`` function is used to read the data for each frame. The read frames, stored in a frame container.
"""
**Parameters:** if self.compressed_io:
return utils.load_compressed(filename, self.preprocessor.read_data)
filename : str else:
The name of the preprocessed data file. return utils.FrameContainer(bob.io.base.HDF5File(filename), self.preprocessor.read_data)
**Returns:**
frames : :py:class:`bob.bio.video.FrameContainer`
The read frames, stored in a frame container.
"""
if self.compressed_io:
return utils.load_compressed(filename, self.preprocessor.read_data)
else:
return utils.FrameContainer(bob.io.base.HDF5File(filename), self.preprocessor.read_data)
def write_data(self, frames, filename):
"""Writes the preprocessed data to file.
def write_data(self, frames, filename): The preprocessors ``write_data`` function is used to write the data for each frame.
"""Writes the preprocessed data to file.
The preprocessors ``write_data`` function is used to write the data for each frame. **Parameters:**
**Parameters:** frames : :py:class:`bob.bio.video.FrameContainer`
The preprocessed frames, as returned by the :py:meth:`__call__` function.
frames : :py:class:`bob.bio.video.FrameContainer` filename : str
The preprocessed frames, as returned by the :py:meth:`__call__` function. The name of the preprocessed data file to write.
"""
self._check_data(frames)
filename : str if self.compressed_io:
The name of the preprocessed data file to write. return utils.save_compressed(frames, filename, self.preprocessor.write_data)
""" else:
self._check_data(frames) frames.save(bob.io.base.HDF5File(filename, 'w'), self.preprocessor.write_data)
if self.compressed_io:
return utils.save_compressed(frames, filename, self.preprocessor.write_data)
else:
frames.save(bob.io.base.HDF5File(filename, 'w'), self.preprocessor.write_data)
from bob.bio.base.database import ZTBioDatabase from bob.bio.base.database import ZTBioDatabase
from bob.bio.base.test.utils import atnt_database_directory from bob.bio.base.test.utils import atnt_database_directory
from bob.bio.base.database.file import BioFile
from bob.bio.video.database import VideoBioFile
from bob.bio.video.utils import FrameContainer
import bob.io.base
import os
class DummyBioFile(VideoBioFile):
def load(self, directory=None, extension='.avi'):
file_name = self.make_path(directory, extension)
fc = FrameContainer()
fc.add(os.path.basename(file_name), bob.io.base.load(file_name))
return fc
class DummyDatabase(ZTBioDatabase): class DummyDatabase(ZTBioDatabase):
...@@ -17,11 +30,14 @@ class DummyDatabase(ZTBioDatabase): ...@@ -17,11 +30,14 @@ class DummyDatabase(ZTBioDatabase):
import bob.db.atnt import bob.db.atnt
self.__db = bob.db.atnt.Database() self.__db = bob.db.atnt.Database()
def _make_bio(self, files):
return [DummyBioFile(BioFile(client_id=f.client_id, path=f.path, file_id=f.id)) for f in files]
def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs): def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
return self.__db.model_ids(groups, protocol) return self.__db.model_ids(groups, protocol)
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs): def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
return self.__db.objects(model_ids, groups, purposes, protocol, **kwargs) return self._make_bio(self.__db.objects(model_ids, groups, purposes, protocol, **kwargs))
def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs): def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
return [] return []
...@@ -39,8 +55,9 @@ class DummyDatabase(ZTBioDatabase): ...@@ -39,8 +55,9 @@ class DummyDatabase(ZTBioDatabase):
return self.probe_files(None, group) return self.probe_files(None, group)
# override all_files to return a one-element lists of files # override all_files to return a one-element lists of files
def all_files(self, groups): #def all_files(self, groups):
return [[n] for n in super(DummyDatabase, self).all_files(groups)] #return [[n] for n in super(DummyDatabase, self).all_files(groups)]
#return [[n] for n in super(DummyDatabase, self).all_files(groups)]
def file_names(self, files, directory, extension): def file_names(self, files, directory, extension):
if isinstance(files[0], list): if isinstance(files[0], list):
......
...@@ -9,70 +9,88 @@ import bob.io.video ...@@ -9,70 +9,88 @@ import bob.io.video
import bob.bio.base import bob.bio.base
import bob.bio.video import bob.bio.video
import bob.db.base import bob.db.base
import pkg_resources import pkg_resources
from bob.bio.video.test.dummy.database import DummyBioFile
regenerate_refs = False regenerate_refs = False
def test_annotations(): def test_annotations():
# use annotations to grep original_path = pkg_resources.resource_filename("bob.bio.face.test", "")
image_files = [pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.jpg")] image_files = DummyBioFile(bob.bio.base.database.BioFile(client_id=1, file_id=1, path="data/testimage"))
annotations = {os.path.basename(image_files[0]) : bob.db.base.read_annotation_file(pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.pos"), 'named')}
# use annotations to grep
annotations = {os.path.basename(image_files.make_path(original_path, ".jpg")): bob.db.base.read_annotation_file(
pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.pos"), 'named')}
# video preprocessor using a face crop preprocessor # video preprocessor using a face crop preprocessor
frame_selector = bob.bio.video.FrameSelector(selection_style="all") frame_selector = bob.bio.video.FrameSelector(selection_style="all")
preprocessor = bob.bio.video.preprocessor.Wrapper('face-crop-eyes', frame_selector, compressed_io=False) preprocessor = bob.bio.video.preprocessor.Wrapper('face-crop-eyes', frame_selector, compressed_io=False)
# read original data # read original data
original = preprocessor.read_original_data(image_files) original = preprocessor.read_original_data(image_files, original_path, ".jpg")
assert isinstance(original, bob.bio.video.FrameContainer)
assert len(original) == 1
assert original[0][0] == os.path.basename(image_files[0])
# preprocess data including annotations assert isinstance(original, bob.bio.video.FrameContainer)
preprocessed = preprocessor(original, annotations) assert len(original) == 1
assert isinstance(preprocessed, bob.bio.video.FrameContainer) assert original[0][0] == os.path.basename(image_files.make_path(original_path, ".jpg"))
assert len(preprocessed) == 1
assert preprocessed[0][0] == os.path.basename(image_files[0])
assert preprocessed[0][2] is None
assert numpy.allclose(preprocessed[0][1], bob.io.base.load(pkg_resources.resource_filename("bob.bio.face.test", "data/cropped.hdf5")))
# preprocess data including annotations
preprocessed = preprocessor(original, annotations)
assert isinstance(preprocessed, bob.bio.video.FrameContainer)
assert len(preprocessed) == 1
assert preprocessed[0][0] == os.path.basename(image_files.make_path(original_path, ".jpg"))
assert preprocessed[0][2] is None
assert numpy.allclose(preprocessed[0][1],
bob.io.base.load(pkg_resources.resource_filename("bob.bio.face.test", "data/cropped.hdf5")))
def test_detect(): def test_detect():
# load test video
video_file = pkg_resources.resource_filename("bob.bio.video.test", "data/testvideo.avi")
frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
preprocessor = bob.bio.video.preprocessor.Wrapper('face-detect', frame_selector, compressed_io=False) def read_original_data(video_object, original_path, original_extension):
video = preprocessor.read_original_data(video_file) return frame_selector(video_object.make_path(original_path, original_extension))
assert isinstance(video, bob.bio.video.FrameContainer)
# load test video
original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
video_object = bob.bio.video.database.VideoBioFile(bob.bio.base.database.BioFile(
client_id=1, file_id=1, path="data/testvideo"))
preprocessed_video = preprocessor(video) frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
assert isinstance(preprocessed_video, bob.bio.video.FrameContainer) preprocessor = bob.bio.video.preprocessor.Wrapper('face-detect', frame_selector, compressed_io=False,
read_original_data=read_original_data)
reference_file = pkg_resources.resource_filename("bob.bio.video.test", "data/preprocessed.hdf5") video = preprocessor.read_original_data(video_object, original_path,".avi")
if regenerate_refs: assert isinstance(video, bob.bio.video.FrameContainer)
preprocessed_video.save(bob.io.base.HDF5File(reference_file, 'w'))
reference_data = bob.bio.video.FrameContainer(bob.io.base.HDF5File(reference_file, 'r'))