Commit eb286c33 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'frame-container-load-subset' into 'master'

Improve FrameContainer and FrameSelector to be more memory efficient

See merge request !40
parents fe242495 b014e4a5
Pipeline #36898 passed with stages
in 12 minutes and 43 seconds
...@@ -9,7 +9,7 @@ class Base(bob.bio.base.annotator.Annotator): ...@@ -9,7 +9,7 @@ class Base(bob.bio.base.annotator.Annotator):
---------- ----------
frame_selector : :any:`bob.bio.video.FrameSelector` frame_selector : :any:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the video to use. A frame selector class to define, which frames of the video to use.
read_original_data : callable read_original_data : ``callable``
A function with the signature of A function with the signature of
``data = read_original_data(biofile, directory, extension)`` ``data = read_original_data(biofile, directory, extension)``
that will be used to load the data from biofiles. By default the that will be used to load the data from biofiles. By default the
......
...@@ -28,7 +28,7 @@ class FailSafeVideo(Base): ...@@ -28,7 +28,7 @@ class FailSafeVideo(Base):
The maximum number of frames that an annotation is valid for next frames. The maximum number of frames that an annotation is valid for next frames.
This value should be positive. If you want to set max_age to infinite, This value should be positive. If you want to set max_age to infinite,
then you can use the :any:`bob.bio.video.annotator.Wrapper` instead. then you can use the :any:`bob.bio.video.annotator.Wrapper` instead.
validator : callable validator : ``callable``
A function that takes the annotations of a frame and validates it. A function that takes the annotations of a frame and validates it.
......
...@@ -10,7 +10,7 @@ def normalize_annotations(annotations, validator, max_age=-1): ...@@ -10,7 +10,7 @@ def normalize_annotations(annotations, validator, max_age=-1):
strings (starting from 0). The inside dicts contain annotations for that strings (starting from 0). The inside dicts contain annotations for that
frame. The dictionary needs to be an ordered dict in order for this to frame. The dictionary needs to be an ordered dict in order for this to
work. work.
validator : callable validator : ``callable``
Takes a dict (annotations) and returns True if the annotations are valid. Takes a dict (annotations) and returns True if the annotations are valid.
This can be a check based on minimal face size for example: see This can be a check based on minimal face size for example: see
:any:`bob.bio.face.annotator.min_face_size_validator`. :any:`bob.bio.face.annotator.min_face_size_validator`.
......
...@@ -41,6 +41,12 @@ def test_frame_container(): ...@@ -41,6 +41,12 @@ def test_frame_container():
# test as_array method # test as_array method
assert numpy.allclose(read.as_array(), test_data) assert numpy.allclose(read.as_array(), test_data)
# check loading only a part of the hdf5
with bob.io.base.HDF5File(filename) as f:
# load only a subset of the FrameContainer
fc = bob.bio.video.FrameContainer().load(f, selection_style="spread", max_number_of_frames=10)
assert len(fc) == 10, len(fc)
finally: finally:
if os.path.exists(filename): if os.path.exists(filename):
os.remove(filename) os.remove(filename)
...@@ -113,3 +119,9 @@ def test_frame_selector(): ...@@ -113,3 +119,9 @@ def test_frame_selector():
assert frames[1][0] == '6' assert frames[1][0] == '6'
assert numpy.allclose(frames[1][1], video_data[6]) assert numpy.allclose(frames[1][1], video_data[6])
assert frames[1][2] is None assert frames[1][2] is None
# test bob.io.video.reader support
path = bob.io.base.test_utils.datafile("testvideo.avi", __name__)
fs = bob.bio.video.FrameSelector(selection_style="spread", max_number_of_frames=20)
fc = fs(bob.io.video.reader(path)) # only loads 20 frames into memory
assert len(fc) == 20, len(fc)
...@@ -3,116 +3,227 @@ ...@@ -3,116 +3,227 @@
import bob.bio.base import bob.bio.base
import numpy import numpy
import logging import logging
logger = logging.getLogger("bob.bio.video")
class FrameContainer: logger = logging.getLogger(__name__)
"""A class for reading, manipulating and saving video content.
"""
def select_frames(count, max_number_of_frames, selection_style, step_size):
def __init__(self, hdf5 = None, load_function = bob.bio.base.load): """Returns indices of the frames to be selected given the parameters.
self._frames = []
if hdf5 is not None: Different selection styles are supported:
self.load(hdf5, load_function)
* first : The first frames are selected
def __len__(self): * spread : Frames are selected to be taken from the whole video with equal spaces in
return len(self._frames) between.
* step : Frames are selected every ``step_size`` indices, starting at
def __iter__(self): ``step_size/2`` **Think twice if you want to have that when giving FrameContainer
"""Generator that returns the 3-tuple (frame_id, data, quality) for each frame.""" data!**
# don't sort * all : All frames are selected unconditionally.
for frame in self._frames:
yield frame Parameters
----------
def __getitem__(self, i): count : int
"""Indexer (mostly used in tests).""" Total number of frames that are available
return self._frames[i] max_number_of_frames : int
The maximum number of frames to be selected. Ignored when selection_style is
def add(self, frame_id, frame, quality = None): "all".
"""Adds the frame with the given id and the given quality.""" selection_style : str
self._frames.append((str(frame_id), frame, quality)) One of (``first``, ``spread``, ``step``, ``all``). See above.
step_size : int
def load(self, hdf5, load_function = bob.bio.base.load): Only useful when ``selection_style`` is ``step``.
self._frames = []
if hdf5.has_group("FrameIndexes"):
hdf5.cd("FrameIndexes")
indices = sorted(int(i) for i in hdf5.keys(relative=True))
frame_ids = [hdf5[str(i)] for i in indices]
hdf5.cd("..")
else:
frame_ids = hdf5.sub_groups(relative=True, recursive=False)
# Read content (frames) from HDF5File
for path in frame_ids:
# extract frame_id
if path[:6] == 'Frame_':
frame_id = str(path[6:])
hdf5.cd(path)
# Read data
data = load_function(hdf5)
# read quality, if present
quality = hdf5.read("FrameQuality") if hdf5.has_key("FrameQuality") else None
self.add(frame_id, data, quality)
hdf5.cd("..")
if not len(self):
raise IOError("Could not load data as a Frame Container from file %s" % hdf5.filename)
def save(self, hdf5, save_function = bob.bio.base.save):
""" Save the content to the given HDF5 File.
The contained data will be written using the given save_function."""
if not len(self):
logger.warn("Saving empty FrameContainer '%s'", hdf5.filename)
frame_ids = []
for frame_id, data, quality in self:
hdf5.create_group("Frame_%s" % frame_id)
hdf5.cd("Frame_%s" % frame_id)
frame_ids.append("Frame_%s" % frame_id)
save_function(data, hdf5)
if quality is not None:
hdf5.set("FrameQuality", quality)
hdf5.cd("..")
# save the order of frames too so we can load them correctly later
hdf5.create_group("FrameIndexes")
hdf5.cd("FrameIndexes")
for i, v in enumerate(frame_ids):
hdf5[str(i)] = v
hdf5.cd("..")
def is_similar_to(self, other):
if len(self) != len(other): return False
for a,b in zip(self, other):
if a[0] != b[0]: return False
if abs(a[2] - b[2]) > 1e-8: return False
if not numpy.allclose(a[1], b[1]): return False
return True
def as_array(self):
"""Returns the data of frames as a numpy array.
Returns Returns
------- -------
numpy.ndarray range
The frames are returned as an array with the shape of (n_frames, ...) A range of frames to be selected.
like a video.
Raises
------
ValueError
If ``selection_style`` is not one of the supported ones.
""" """
def _reader(frame): if selection_style == "first":
# Each frame is assumed to be an image here. We make it a single frame # get the first frames (limited by all frames)
# video here by expanding its dimensions. This way it can be used with indices = range(0, min(count, max_number_of_frames))
# the vstack_features function. elif selection_style == "spread":
return frame[1][None, ...] # get frames lineraly spread over all frames
return bob.bio.base.vstack_features(_reader, self._frames, same_size=True) indices = bob.bio.base.selected_indices(count, max_number_of_frames)
elif selection_style == "step":
indices = range(step_size // 2, count, step_size)[:max_number_of_frames]
elif selection_style == "all":
indices = range(0, count)
else:
raise ValueError(f"Invalid selection style: {selection_style}")
return indices
class FrameContainer:
"""A class for reading, manipulating and saving video content.
"""
def __init__(self, hdf5=None, load_function=bob.bio.base.load, **kwargs):
super().__init__(**kwargs)
self._frames = []
if hdf5 is not None:
self.load(hdf5, load_function)
def __len__(self):
return len(self._frames)
def __iter__(self):
"""Generator that returns the 3-tuple (frame_id, data, quality) for each frame."""
# don't sort
for frame in self._frames:
yield frame
def __getitem__(self, i):
"""Indexer (mostly used in tests)."""
return self._frames[i]
def add(self, frame_id, frame, quality=None):
"""Adds the frame with the given id and the given quality."""
self._frames.append((str(frame_id), frame, quality))
def load(
self,
hdf5,
load_function=bob.bio.base.load,
selection_style="all",
max_number_of_frames=20,
step_size=10,
):
"""Loads a previously saved FrameContainer into the current FrameContainer.
Parameters
----------
hdf5 : :any:`bob.io.base.HDF5File`
An opened HDF5 file to load the data form
load_function : ``callable``, ``optional``
the function to be used on the hdf5 object to load each frame
selection_style : str, ``optional``
See :any:`select_frames`
max_number_of_frames : int, ``optional``
See :any:`select_frames`
step_size : int, ``optional``
See :any:`select_frames`
Returns
-------
object
returns itself.
Raises
------
IOError
If no frames can be loaded from the hdf5 file.
ValueError
If the selection_style is all and you are trying to load an old format
FrameContainer.
"""
self._frames = []
if hdf5.has_group("FrameIndexes"):
hdf5.cd("FrameIndexes")
indices = sorted(int(i) for i in hdf5.keys(relative=True))
indices = select_frames(
count=len(indices),
max_number_of_frames=max_number_of_frames,
selection_style=selection_style,
step_size=step_size,
)
frame_ids = [hdf5[str(i)] for i in indices]
hdf5.cd("..")
else:
if selection_style != "all":
raise ValueError(
"selection_style must be all when loading FrameContainers with "
"the old format. Try re-writing the FrameContainers again "
"to avoid this."
)
frame_ids = hdf5.sub_groups(relative=True, recursive=False)
# Read content (frames) from HDF5File
for path in frame_ids:
# extract frame_id
if path[:6] == "Frame_":
frame_id = str(path[6:])
hdf5.cd(path)
# Read data
data = load_function(hdf5)
# read quality, if present
quality = hdf5["FrameQuality"] if "FrameQuality" in hdf5 else None
self.add(frame_id, data, quality)
hdf5.cd("..")
if not len(self):
raise IOError(
"Could not load data as a Frame Container from file %s" % hdf5.filename
)
return self
def save(self, hdf5, save_function=bob.bio.base.save):
""" Save the content to the given HDF5 File.
The contained data will be written using the given save_function."""
if not len(self):
logger.warn("Saving empty FrameContainer '%s'", hdf5.filename)
frame_ids = []
for frame_id, data, quality in self:
hdf5.create_group("Frame_%s" % frame_id)
hdf5.cd("Frame_%s" % frame_id)
frame_ids.append("Frame_%s" % frame_id)
save_function(data, hdf5)
if quality is not None:
hdf5.set("FrameQuality", quality)
hdf5.cd("..")
# save the order of frames too so we can load them correctly later
hdf5.create_group("FrameIndexes")
hdf5.cd("FrameIndexes")
for i, v in enumerate(frame_ids):
hdf5[str(i)] = v
hdf5.cd("..")
def is_similar_to(self, other):
if len(self) != len(other):
return False
for a, b in zip(self, other):
if a[0] != b[0]:
return False
if abs(a[2] - b[2]) > 1e-8:
return False
if not numpy.allclose(a[1], b[1]):
return False
return True
def as_array(self):
"""Returns the data of frames as a numpy array.
Returns
-------
numpy.ndarray
The frames are returned as an array with the shape of (n_frames, ...)
like a video.
"""
def _reader(frame):
# Each frame is assumed to be an image here. We make it a single frame
# video here by expanding its dimensions. This way it can be used with
# the vstack_features function.
return frame[1][None, ...]
return bob.bio.base.vstack_features(_reader, self._frames, same_size=True)
def save_compressed(frame_container, filename, save_function, create_link=True): def save_compressed(frame_container, filename, save_function, create_link=True):
hdf5 = bob.bio.base.open_compressed(filename, 'w') hdf5 = bob.bio.base.open_compressed(filename, "w")
frame_container.save(hdf5, save_function) frame_container.save(hdf5, save_function)
bob.bio.base.close_compressed(filename, hdf5, create_link=create_link) bob.bio.base.close_compressed(filename, hdf5, create_link=create_link)
del hdf5 del hdf5
def load_compressed(filename, load_function): def load_compressed(filename, load_function):
hdf5 = bob.bio.base.open_compressed(filename, 'r') hdf5 = bob.bio.base.open_compressed(filename, "r")
fc = FrameContainer(hdf5, load_function) fc = FrameContainer(hdf5, load_function)
bob.bio.base.close_compressed(filename, hdf5) bob.bio.base.close_compressed(filename, hdf5)
del hdf5 del hdf5
return fc return fc
...@@ -10,84 +10,93 @@ import os ...@@ -10,84 +10,93 @@ import os
import six import six
import logging import logging
logger = logging.getLogger("bob.bio.video")
from .FrameContainer import FrameContainer logger = logging.getLogger(__name__)
from .FrameContainer import FrameContainer, select_frames
class FrameSelector: class FrameSelector:
"""A class for selecting frames from videos. """A class for selecting frames from videos.
In total, up to ``max_number_of_frames`` is selected (unless selection style is ``all`` In total, up to ``max_number_of_frames`` is selected (unless selection style is ``all``
Different selection styles are supported: Different selection styles are supported:
* first : The first frames are selected * first : The first frames are selected
* spread : Frames are selected to be taken from the whole video * spread : Frames are selected to be taken from the whole video
* step : Frames are selected every ``step_size`` indices, starting at ``step_size/2`` **Think twice if you want to have that when giving FrameContainer data!** * step : Frames are selected every ``step_size`` indices, starting at ``step_size/2`` **Think twice if you want to have that when giving FrameContainer data!**
* all : All frames are stored unconditionally * all : All frames are stored unconditionally
* quality (only valid for FrameContainer data) : Select the frames based on the highest internally stored quality value * quality (only valid for FrameContainer data) : Select the frames based on the highest internally stored quality value
"""
def __init__(self,
max_number_of_frames = 20,
selection_style = "spread",
step_size = 10
):
if selection_style not in ('first', 'spread', 'step', 'all'):
raise ValueError("Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')" % selection_style)
self.selection = selection_style
self.max_frames = max_number_of_frames
self.step = step_size
def __call__(self, data, load_function = bob.io.base.load):
"""Selects frames and returns them in a FrameContainer.
Different ``data`` parameters are accepted:
* :py:class:`FrameContainer` : frames are selected from the given frame container
* ``str`` : A video file to read and select frames from
* ``[str]`` : A list of image names to select from
* ``numpy.array`` (3D or 4D): A video to select frames from
When giving ``str`` or ``[str]`` data, the given ``load_function`` is used to read the data from file.
""" """
# if given a string, first load the video
if isinstance(data, six.string_types): def __init__(self, max_number_of_frames=20, selection_style="spread", step_size=10):
logger.debug("Loading video file '%s'", data) if selection_style not in ("first", "spread", "step", "all"):
data = load_function(data) raise ValueError(
"Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')"
# first, get the indices % selection_style
count = len(data) )
if self.selection == 'first': self.selection = selection_style
# get the first frames (limited by all frames) self.max_frames = max_number_of_frames
indices = range(0, min(count, self.max_frames)) self.step = step_size
elif self.selection == 'spread':
# get frames lineraly spread over all frames def __call__(self, data, load_function=bob.io.base.load):
indices = bob.bio.base.selected_indices(count, self.max_frames) """Selects frames and returns them in a FrameContainer.
elif self.selection == 'step': Different ``data`` parameters are accepted:
indices = range(self.step//2, count, self.step)[:self.max_frames]
elif self.selection == 'all': * :py:class:`FrameContainer` : frames are selected from the given frame container
indices = range(0, count) * ``str`` : A video file to read and select frames from
* ``[str]`` : A list of image names to select from
# now, iterate through the data * ``numpy.array`` (3D or 4D): A video to select frames from
fc = FrameContainer() * ``bob.io.video.reader`` : An instance of bob.io.video.reader.
if isinstance(data, FrameContainer):
indices = set(indices) When giving ``str`` or ``[str]`` data, the given ``load_function`` is used to read the data from file.
# frame container data, just copy """
for i, frame in enumerate(data): # if given a string, first load the video
if i in indices: if isinstance(data, six.string_types):
fc.add(*frame) logger.debug("Loading video file '%s'", data)
elif isinstance(data, numpy.ndarray): data = load_function(data)
# select video frames
for i in indices: # first, get the indices
fc.add(i, data[i]) if isinstance(data, bob.io.video.reader):
elif isinstance(data, list): count = data.number_of_frames
for i in indices: else:
# load image count = len(data)
image = load_function(data[i])
# save image name as well indices = select_frames(
fc.add(os.path.basename(data[i]), image) count=count,
max_number_of_frames=self.max_frames,
return fc selection_style=self.selection,
step_size=self.step,
def __str__(self): )
"""Writes the parameters of the FrameSelector as a string."""
return "FrameSelector(max_number_of_frames=%d, selection_style='%s', step_size=%d)" % (self.max_frames, self.selection, self.step) # now, iterate through the data
fc = FrameContainer()
if isinstance(data, FrameContainer):
indices = set(indices)
# frame container data, just copy
for i, frame in enumerate(data):
if i in indices:
fc.add(*frame)
elif isinstance(data, bob.io.video.reader):
for i, frame in enumerate(data):
if i in indices:
fc.add(i, frame)
elif isinstance(data, numpy.ndarray):
# select video frames
for i in indices:
fc.add(i, data[i])
elif isinstance(data, list):
for i in indices:
# load image
image = load_function(data[i])
# save image name as well
fc.add(os.path.basename(data[i]), image)
return fc
def __str__(self):
"""Writes the parameters of the FrameSelector as a string."""
return (
"FrameSelector(max_number_of_frames=%d, selection_style='%s', step_size=%d)"
% (self.max_frames, self.selection, self.step)
)
from .FrameContainer import FrameContainer, load_compressed, save_compressed from .FrameContainer import FrameContainer, load_compressed, save_compressed, select_frames
from .FrameSelector import FrameSelector from .FrameSelector import FrameSelector
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment