Commit eb286c33 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira

Merge branch 'frame-container-load-subset' into 'master'

Improve FrameContainer and FrameSelector to be more memory efficient

See merge request !40
parents fe242495 b014e4a5
Pipeline #36898 passed with stages
in 12 minutes and 43 seconds
...@@ -9,7 +9,7 @@ class Base(bob.bio.base.annotator.Annotator): ...@@ -9,7 +9,7 @@ class Base(bob.bio.base.annotator.Annotator):
---------- ----------
frame_selector : :any:`bob.bio.video.FrameSelector` frame_selector : :any:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the video to use. A frame selector class to define, which frames of the video to use.
read_original_data : callable read_original_data : ``callable``
A function with the signature of A function with the signature of
``data = read_original_data(biofile, directory, extension)`` ``data = read_original_data(biofile, directory, extension)``
that will be used to load the data from biofiles. By default the that will be used to load the data from biofiles. By default the
......
...@@ -28,7 +28,7 @@ class FailSafeVideo(Base): ...@@ -28,7 +28,7 @@ class FailSafeVideo(Base):
The maximum number of frames that an annotation is valid for next frames. The maximum number of frames that an annotation is valid for next frames.
This value should be positive. If you want to set max_age to infinite, This value should be positive. If you want to set max_age to infinite,
then you can use the :any:`bob.bio.video.annotator.Wrapper` instead. then you can use the :any:`bob.bio.video.annotator.Wrapper` instead.
validator : callable validator : ``callable``
A function that takes the annotations of a frame and validates it. A function that takes the annotations of a frame and validates it.
......
...@@ -10,7 +10,7 @@ def normalize_annotations(annotations, validator, max_age=-1): ...@@ -10,7 +10,7 @@ def normalize_annotations(annotations, validator, max_age=-1):
strings (starting from 0). The inside dicts contain annotations for that strings (starting from 0). The inside dicts contain annotations for that
frame. The dictionary needs to be an ordered dict in order for this to frame. The dictionary needs to be an ordered dict in order for this to
work. work.
validator : callable validator : ``callable``
Takes a dict (annotations) and returns True if the annotations are valid. Takes a dict (annotations) and returns True if the annotations are valid.
This can be a check based on minimal face size for example: see This can be a check based on minimal face size for example: see
:any:`bob.bio.face.annotator.min_face_size_validator`. :any:`bob.bio.face.annotator.min_face_size_validator`.
......
...@@ -41,6 +41,12 @@ def test_frame_container(): ...@@ -41,6 +41,12 @@ def test_frame_container():
# test as_array method # test as_array method
assert numpy.allclose(read.as_array(), test_data) assert numpy.allclose(read.as_array(), test_data)
# check loading only a part of the hdf5
with bob.io.base.HDF5File(filename) as f:
# load only a subset of the FrameContainer
fc = bob.bio.video.FrameContainer().load(f, selection_style="spread", max_number_of_frames=10)
assert len(fc) == 10, len(fc)
finally: finally:
if os.path.exists(filename): if os.path.exists(filename):
os.remove(filename) os.remove(filename)
...@@ -113,3 +119,9 @@ def test_frame_selector(): ...@@ -113,3 +119,9 @@ def test_frame_selector():
assert frames[1][0] == '6' assert frames[1][0] == '6'
assert numpy.allclose(frames[1][1], video_data[6]) assert numpy.allclose(frames[1][1], video_data[6])
assert frames[1][2] is None assert frames[1][2] is None
# test bob.io.video.reader support
path = bob.io.base.test_utils.datafile("testvideo.avi", __name__)
fs = bob.bio.video.FrameSelector(selection_style="spread", max_number_of_frames=20)
fc = fs(bob.io.video.reader(path)) # only loads 20 frames into memory
assert len(fc) == 20, len(fc)
...@@ -3,15 +3,68 @@ ...@@ -3,15 +3,68 @@
import bob.bio.base import bob.bio.base
import numpy import numpy
import logging import logging
logger = logging.getLogger("bob.bio.video")
logger = logging.getLogger(__name__)
def select_frames(count, max_number_of_frames, selection_style, step_size):
"""Returns indices of the frames to be selected given the parameters.
Different selection styles are supported:
* first : The first frames are selected
* spread : Frames are selected to be taken from the whole video with equal spaces in
between.
* step : Frames are selected every ``step_size`` indices, starting at
``step_size/2`` **Think twice if you want to have that when giving FrameContainer
data!**
* all : All frames are selected unconditionally.
Parameters
----------
count : int
Total number of frames that are available
max_number_of_frames : int
The maximum number of frames to be selected. Ignored when selection_style is
"all".
selection_style : str
One of (``first``, ``spread``, ``step``, ``all``). See above.
step_size : int
Only useful when ``selection_style`` is ``step``.
Returns
-------
range
A range of frames to be selected.
Raises
------
ValueError
If ``selection_style`` is not one of the supported ones.
"""
if selection_style == "first":
# get the first frames (limited by all frames)
indices = range(0, min(count, max_number_of_frames))
elif selection_style == "spread":
# get frames lineraly spread over all frames
indices = bob.bio.base.selected_indices(count, max_number_of_frames)
elif selection_style == "step":
indices = range(step_size // 2, count, step_size)[:max_number_of_frames]
elif selection_style == "all":
indices = range(0, count)
else:
raise ValueError(f"Invalid selection style: {selection_style}")
return indices
class FrameContainer: class FrameContainer:
"""A class for reading, manipulating and saving video content. """A class for reading, manipulating and saving video content.
""" """
def __init__(self, hdf5 = None, load_function = bob.bio.base.load): def __init__(self, hdf5=None, load_function=bob.bio.base.load, **kwargs):
super().__init__(**kwargs)
self._frames = [] self._frames = []
if hdf5 is not None: if hdf5 is not None:
self.load(hdf5, load_function) self.load(hdf5, load_function)
...@@ -29,37 +82,87 @@ class FrameContainer: ...@@ -29,37 +82,87 @@ class FrameContainer:
"""Indexer (mostly used in tests).""" """Indexer (mostly used in tests)."""
return self._frames[i] return self._frames[i]
def add(self, frame_id, frame, quality = None): def add(self, frame_id, frame, quality=None):
"""Adds the frame with the given id and the given quality.""" """Adds the frame with the given id and the given quality."""
self._frames.append((str(frame_id), frame, quality)) self._frames.append((str(frame_id), frame, quality))
def load(self, hdf5, load_function = bob.bio.base.load): def load(
self,
hdf5,
load_function=bob.bio.base.load,
selection_style="all",
max_number_of_frames=20,
step_size=10,
):
"""Loads a previously saved FrameContainer into the current FrameContainer.
Parameters
----------
hdf5 : :any:`bob.io.base.HDF5File`
An opened HDF5 file to load the data form
load_function : ``callable``, ``optional``
the function to be used on the hdf5 object to load each frame
selection_style : str, ``optional``
See :any:`select_frames`
max_number_of_frames : int, ``optional``
See :any:`select_frames`
step_size : int, ``optional``
See :any:`select_frames`
Returns
-------
object
returns itself.
Raises
------
IOError
If no frames can be loaded from the hdf5 file.
ValueError
If the selection_style is all and you are trying to load an old format
FrameContainer.
"""
self._frames = [] self._frames = []
if hdf5.has_group("FrameIndexes"): if hdf5.has_group("FrameIndexes"):
hdf5.cd("FrameIndexes") hdf5.cd("FrameIndexes")
indices = sorted(int(i) for i in hdf5.keys(relative=True)) indices = sorted(int(i) for i in hdf5.keys(relative=True))
indices = select_frames(
count=len(indices),
max_number_of_frames=max_number_of_frames,
selection_style=selection_style,
step_size=step_size,
)
frame_ids = [hdf5[str(i)] for i in indices] frame_ids = [hdf5[str(i)] for i in indices]
hdf5.cd("..") hdf5.cd("..")
else: else:
if selection_style != "all":
raise ValueError(
"selection_style must be all when loading FrameContainers with "
"the old format. Try re-writing the FrameContainers again "
"to avoid this."
)
frame_ids = hdf5.sub_groups(relative=True, recursive=False) frame_ids = hdf5.sub_groups(relative=True, recursive=False)
# Read content (frames) from HDF5File # Read content (frames) from HDF5File
for path in frame_ids: for path in frame_ids:
# extract frame_id # extract frame_id
if path[:6] == 'Frame_': if path[:6] == "Frame_":
frame_id = str(path[6:]) frame_id = str(path[6:])
hdf5.cd(path) hdf5.cd(path)
# Read data # Read data
data = load_function(hdf5) data = load_function(hdf5)
# read quality, if present # read quality, if present
quality = hdf5.read("FrameQuality") if hdf5.has_key("FrameQuality") else None quality = hdf5["FrameQuality"] if "FrameQuality" in hdf5 else None
self.add(frame_id, data, quality) self.add(frame_id, data, quality)
hdf5.cd("..") hdf5.cd("..")
if not len(self): if not len(self):
raise IOError("Could not load data as a Frame Container from file %s" % hdf5.filename) raise IOError(
"Could not load data as a Frame Container from file %s" % hdf5.filename
)
return self
def save(self, hdf5, save_function = bob.bio.base.save): def save(self, hdf5, save_function=bob.bio.base.save):
""" Save the content to the given HDF5 File. """ Save the content to the given HDF5 File.
The contained data will be written using the given save_function.""" The contained data will be written using the given save_function."""
if not len(self): if not len(self):
...@@ -81,11 +184,15 @@ class FrameContainer: ...@@ -81,11 +184,15 @@ class FrameContainer:
hdf5.cd("..") hdf5.cd("..")
def is_similar_to(self, other): def is_similar_to(self, other):
if len(self) != len(other): return False if len(self) != len(other):
for a,b in zip(self, other): return False
if a[0] != b[0]: return False for a, b in zip(self, other):
if abs(a[2] - b[2]) > 1e-8: return False if a[0] != b[0]:
if not numpy.allclose(a[1], b[1]): return False return False
if abs(a[2] - b[2]) > 1e-8:
return False
if not numpy.allclose(a[1], b[1]):
return False
return True return True
def as_array(self): def as_array(self):
...@@ -97,21 +204,25 @@ class FrameContainer: ...@@ -97,21 +204,25 @@ class FrameContainer:
The frames are returned as an array with the shape of (n_frames, ...) The frames are returned as an array with the shape of (n_frames, ...)
like a video. like a video.
""" """
def _reader(frame): def _reader(frame):
# Each frame is assumed to be an image here. We make it a single frame # Each frame is assumed to be an image here. We make it a single frame
# video here by expanding its dimensions. This way it can be used with # video here by expanding its dimensions. This way it can be used with
# the vstack_features function. # the vstack_features function.
return frame[1][None, ...] return frame[1][None, ...]
return bob.bio.base.vstack_features(_reader, self._frames, same_size=True) return bob.bio.base.vstack_features(_reader, self._frames, same_size=True)
def save_compressed(frame_container, filename, save_function, create_link=True): def save_compressed(frame_container, filename, save_function, create_link=True):
hdf5 = bob.bio.base.open_compressed(filename, 'w') hdf5 = bob.bio.base.open_compressed(filename, "w")
frame_container.save(hdf5, save_function) frame_container.save(hdf5, save_function)
bob.bio.base.close_compressed(filename, hdf5, create_link=create_link) bob.bio.base.close_compressed(filename, hdf5, create_link=create_link)
del hdf5 del hdf5
def load_compressed(filename, load_function): def load_compressed(filename, load_function):
hdf5 = bob.bio.base.open_compressed(filename, 'r') hdf5 = bob.bio.base.open_compressed(filename, "r")
fc = FrameContainer(hdf5, load_function) fc = FrameContainer(hdf5, load_function)
bob.bio.base.close_compressed(filename, hdf5) bob.bio.base.close_compressed(filename, hdf5)
del hdf5 del hdf5
......
...@@ -10,9 +10,11 @@ import os ...@@ -10,9 +10,11 @@ import os
import six import six
import logging import logging
logger = logging.getLogger("bob.bio.video")
from .FrameContainer import FrameContainer logger = logging.getLogger(__name__)
from .FrameContainer import FrameContainer, select_frames
class FrameSelector: class FrameSelector:
"""A class for selecting frames from videos. """A class for selecting frames from videos.
...@@ -27,18 +29,17 @@ class FrameSelector: ...@@ -27,18 +29,17 @@ class FrameSelector:
* quality (only valid for FrameContainer data) : Select the frames based on the highest internally stored quality value * quality (only valid for FrameContainer data) : Select the frames based on the highest internally stored quality value
""" """
def __init__(self, def __init__(self, max_number_of_frames=20, selection_style="spread", step_size=10):
max_number_of_frames = 20, if selection_style not in ("first", "spread", "step", "all"):
selection_style = "spread", raise ValueError(
step_size = 10 "Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')"
): % selection_style
if selection_style not in ('first', 'spread', 'step', 'all'): )
raise ValueError("Unknown selection style '%s', choose one of ('first', 'spread', 'step', 'all')" % selection_style)
self.selection = selection_style self.selection = selection_style
self.max_frames = max_number_of_frames self.max_frames = max_number_of_frames
self.step = step_size self.step = step_size
def __call__(self, data, load_function = bob.io.base.load): def __call__(self, data, load_function=bob.io.base.load):
"""Selects frames and returns them in a FrameContainer. """Selects frames and returns them in a FrameContainer.
Different ``data`` parameters are accepted: Different ``data`` parameters are accepted:
...@@ -46,6 +47,7 @@ class FrameSelector: ...@@ -46,6 +47,7 @@ class FrameSelector:
* ``str`` : A video file to read and select frames from * ``str`` : A video file to read and select frames from
* ``[str]`` : A list of image names to select from * ``[str]`` : A list of image names to select from
* ``numpy.array`` (3D or 4D): A video to select frames from * ``numpy.array`` (3D or 4D): A video to select frames from
* ``bob.io.video.reader`` : An instance of bob.io.video.reader.
When giving ``str`` or ``[str]`` data, the given ``load_function`` is used to read the data from file. When giving ``str`` or ``[str]`` data, the given ``load_function`` is used to read the data from file.
""" """
...@@ -55,17 +57,17 @@ class FrameSelector: ...@@ -55,17 +57,17 @@ class FrameSelector:
data = load_function(data) data = load_function(data)
# first, get the indices # first, get the indices
if isinstance(data, bob.io.video.reader):
count = data.number_of_frames
else:
count = len(data) count = len(data)
if self.selection == 'first':
# get the first frames (limited by all frames) indices = select_frames(
indices = range(0, min(count, self.max_frames)) count=count,
elif self.selection == 'spread': max_number_of_frames=self.max_frames,
# get frames lineraly spread over all frames selection_style=self.selection,
indices = bob.bio.base.selected_indices(count, self.max_frames) step_size=self.step,
elif self.selection == 'step': )
indices = range(self.step//2, count, self.step)[:self.max_frames]
elif self.selection == 'all':
indices = range(0, count)
# now, iterate through the data # now, iterate through the data
fc = FrameContainer() fc = FrameContainer()
...@@ -75,6 +77,10 @@ class FrameSelector: ...@@ -75,6 +77,10 @@ class FrameSelector:
for i, frame in enumerate(data): for i, frame in enumerate(data):
if i in indices: if i in indices:
fc.add(*frame) fc.add(*frame)
elif isinstance(data, bob.io.video.reader):
for i, frame in enumerate(data):
if i in indices:
fc.add(i, frame)
elif isinstance(data, numpy.ndarray): elif isinstance(data, numpy.ndarray):
# select video frames # select video frames
for i in indices: for i in indices:
...@@ -90,4 +96,7 @@ class FrameSelector: ...@@ -90,4 +96,7 @@ class FrameSelector:
def __str__(self): def __str__(self):
"""Writes the parameters of the FrameSelector as a string.""" """Writes the parameters of the FrameSelector as a string."""
return "FrameSelector(max_number_of_frames=%d, selection_style='%s', step_size=%d)" % (self.max_frames, self.selection, self.step) return (
"FrameSelector(max_number_of_frames=%d, selection_style='%s', step_size=%d)"
% (self.max_frames, self.selection, self.step)
)
from .FrameContainer import FrameContainer, load_compressed, save_compressed from .FrameContainer import FrameContainer, load_compressed, save_compressed, select_frames
from .FrameSelector import FrameSelector from .FrameSelector import FrameSelector
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment