Commit 5e950b3c authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Finalize video annotators and add docs and tests

parent baeeb981
Pipeline #17660 passed with stage
in 20 minutes and 24 seconds
import bob.bio.base
from .. import utils
class Base(bob.bio.base.annotator.Annotator):
"""The base class for video annotators.
Parameters
----------
frame_selector : :any:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the video to use.
read_original_data : callable
A function with the signature of
``data = read_original_data(biofile, directory, extension)``
that will be used to load the data from biofiles. By default the
``frame_selector`` is used to load the data.
"""
def __init__(self, frame_selector=utils.FrameSelector(selection_style='all'),
read_original_data=None, **kwargs):
def _read_video_data(biofile, directory, extension):
"""Read video data using the frame_selector of this object"""
return biofile.load(directory, extension, frame_selector)
if read_original_data is None:
read_original_data = _read_video_data
super(Base, self).__init__(read_original_data=read_original_data, **kwargs)
@staticmethod
def frame_and_frame_ids(frames):
"""Normalizes the frames parameter into a numpy array and a list of
indices.
Parameters
----------
frames : :any:`bob.bio.video.FrameContainer` or :any:`numpy.array`
The frames of the video file.
Returns
-------
frames : :any:`numpy.array`
The frames of the video file as a numpy array.
frame_ids : list
A list of strings that represent the frame ids.
"""
if isinstance(frames, utils.FrameContainer):
frame_ids = [fid for fid, _, _ in frames]
frames = frames.as_array()
else: # frames is already a numpy.array
frame_ids = [str(i) for i in range(len(frames))]
return frames, frame_ids
def annotate(self, frames, **kwargs):
"""Annotates videos.
Parameters
----------
frames : :any:`bob.bio.video.FrameContainer` or :any:`numpy.array`
The frames of the video file.
**kwargs
Extra arguments that annotators may need.
Returns
-------
collections.OrderedDict
A dictionary where its key is the frame id as a string and its value
is a dictionary that are the annotations for that frame.
.. note::
You can use the :any:`Base.frame_and_frame_ids` functions to normalize
the input in your implementation.
"""
raise NotImplementedError()
from collections import OrderedDict
from bob.bio.face.annotator import Base, min_face_size_validator
from .. import utils
from . import Base
import bob.bio.face
import collections
import logging
import six
logger = logging.getLogger(__name__)
......@@ -13,37 +14,45 @@ class FailSafeVideo(Base):
:any:`bob.bio.base.annotator.FailSafe` is that this one tries to use
annotations from older frames (if valid) before trying the next annotator.
.. warning::
You must be careful in using this annotator since different annotators
could have different results. For example the bounding box of one
annotator be totally different from another annotator.
Attributes
----------
annotators : list
A list of annotators to try
A list of annotators to try.
max_age : int
The maximum number of frames that an annotation is valid for next frames.
This value should be positive. If you want to set max_age to infinite,
then you can use the :any:`bob.bio.video.annotator.Wrapper` instead.
required_keys : list
A list of keys that should be available in annotations to stop trying
different annotators.
validator : callable
A function that takes the annotations and validates it.
A function that takes the annotations of a frame and validates it.
"""
def __init__(self, annotators, required_keys, max_age=20,
validator=min_face_size_validator, **kwargs):
def __init__(self, annotators, max_age=15,
validator=bob.bio.face.annotator.min_face_size_validator,
**kwargs):
super(FailSafeVideo, self).__init__(**kwargs)
assert max_age > 0, "max_age: `{}' cannot be less than 1".format(max_age)
self.annotators = list(annotators)
self.required_keys = list(required_keys)
self.annotators = []
for annotator in annotators:
if isinstance(annotator, six.string_types):
annotator = bob.bio.base.load_resource(annotator, 'annotator')
self.annotators.append(annotator)
self.max_age = max_age
self.validator = validator
def annotate(self, frames, **kwargs):
if isinstance(frames, utils.FrameContainer):
frames = frames.as_array()
annotations = OrderedDict()
"""See :any:`Base.annotate`
"""
frames, frame_ids = self.frame_and_frame_ids(frames)
annotations = collections.OrderedDict()
current = None
age = 0
for i, frame in enumerate(frames):
for i, frame in zip(frame_ids, frames):
for annotator in self.annotators:
annot = annotator.annotate(frame, **kwargs)
if annot and self.validator(annot):
......
import six
from collections import OrderedDict
from bob.bio.face.annotator import Base
from bob.bio.base import load_resource
from bob.bio.face.annotator import min_face_size_validator
from .. import utils
from . import normalize_annotations
import collections
import bob.bio.base
import bob.bio.face
from . import Base, normalize_annotations
class Wrapper(Base):
"""Annotates video files.
This annotator does not support annotating only select frames of a video.
"""Annotates video files using the provided image annotator.
See the documentation of :any:`Base` too.
Attributes
Parameters
----------
annotator : :any:`Base`
The image annotator to be used.
annotator : :any:`bob.bio.base.annotator.Annotator` or str
The image annotator to be used. The annotator could also be the name of a
bob.bio.annotator resource which will be loaded.
max_age : int
see :any:`normalize_annotations`.
normalize : bool
If True, it will normalize annotations using :any:`normalize_annotations`
validator : object
see :any:`normalize_annotations` and :any:`min_face_size_validator`.
See :any:`normalize_annotations` and
:any:`bob.bio.face.annotator.min_face_size_validator` for one example.
.. warning::
You should only set ``normalize`` to True only if you are annotating
**all** frames of the video file.
"""
def __init__(self,
annotator,
normalize=True,
validator=min_face_size_validator,
normalize=False,
validator=bob.bio.face.annotator.min_face_size_validator,
max_age=-1,
**kwargs
):
......@@ -39,15 +45,16 @@ class Wrapper(Base):
# load annotator configuration
if isinstance(annotator, six.string_types):
self.annotator = load_resource(annotator, "annotator")
self.annotator = bob.bio.base.load_resource(annotator, "annotator")
def annotate(self, frames, **kwargs):
if isinstance(frames, utils.FrameContainer):
frames = frames.as_array()
annotations = OrderedDict()
for i, frame in enumerate(frames):
"""See :any:`Base.annotate`
"""
frames, frame_ids = self.frame_and_frame_ids(frames)
annotations = collections.OrderedDict()
for i, frame in zip(frame_ids, frames):
annotations[str(i)] = self.annotator(frame, **kwargs)
if self.normalize:
annotations = OrderedDict(normalize_annotations(
annotations = collections.OrderedDict(normalize_annotations(
annotations, self.validator, self.max_age))
return annotations
from ..utils import FrameContainer as _FrameContainer
def normalize_annotations(annotations, validator, max_age=-1):
"""Normalizes the annotations of one video sequence. It fills the
annotations for frames from previous ones if the annotation for the current
frame is not valid.
Parameters
----------
annotations : dict
A dict of dict where the keys to the first dict are frame indices as
strings (starting from 0). The inside dicts contain annotations for
that frame.
validator : callable
Takes a dict (annotations) and returns True if the annotations are
valid. This can be a check based on minimal face size for example.
max_age : :obj:`int`, optional
An integer indicating for a how many frames a detected face is valid if
no detection occurs after such frame. A value of -1 == forever
Yields
------
str
The index of frame.
dict
The corrected annotations of the frame.
"""
# the annotations for the current frame
current = None
age = 0
for k, annot in annotations.items():
if annot and validator(annot):
current = annot
age = 0
elif max_age < 0 or age < max_age:
age += 1
else: # no detections and age is larger than maximum allowed
current = None
yield k, current
"""Normalizes the annotations of one video sequence. It fills the
annotations for frames from previous ones if the annotation for the current
frame is not valid.
Parameters
----------
annotations : collections.OrderedDict
A dict of dict where the keys to the first dict are frame indices as
strings (starting from 0). The inside dicts contain annotations for that
frame. The dictionary needs to be an ordered dict in order for this to
work.
validator : callable
Takes a dict (annotations) and returns True if the annotations are valid.
This can be a check based on minimal face size for example.
max_age : :obj:`int`, optional
An integer indicating for a how many frames a detected face is valid if
no detection occurs after such frame. A value of -1 == forever
Yields
------
str
The index of frame.
dict
The corrected annotations of the frame.
"""
# the annotations for the current frame
current = None
age = 0
for k, annot in annotations.items():
if annot and validator(annot):
current = annot
age = 0
elif max_age < 0 or age < max_age:
age += 1
else: # no detections and age is larger than maximum allowed
current = None
yield k, current
# these imports should be here to avoid circular dependencies
from .Base import Base
from .Wrapper import Wrapper
from .FailSafeVideo import FailSafeVideo
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened.
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened.
Parameters
----------
*args
An iterable of objects to modify
Parameters
----------
*args
An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
for obj in args:
obj.__module__ = __name__
__appropriate__(
Base,
Wrapper,
FailSafeVideo,
)
......
import os
import collections
import bob.io.base
import bob.io.image
import bob.io.video
import bob.bio.video
import pkg_resources
from bob.bio.video.test.dummy.database import DummyBioFile
from bob.bio.face.test.test_annotators import _assert_bob_ip_facedetect
class FailSucessAnnotator(bob.bio.base.annotator.Annotator):
"""An annotator that fails for every second time it is called."""
def __init__(self, **kwargs):
super(FailSucessAnnotator, self).__init__(**kwargs)
self.failed_last_time = True
def annotate(self, image, **kwargs):
if not self.failed_last_time:
self.failed_last_time = True
return None
else:
self.failed_last_time = False
return {
'topleft': (0, 0),
'bottomright': (64, 64)
}
def test_wrapper():
original_path = pkg_resources.resource_filename("bob.bio.face.test", "")
image_files = DummyBioFile(client_id=1, file_id=1, path="data/testimage")
# video preprocessor using a face crop preprocessor
annotator = bob.bio.video.annotator.Wrapper('facedetect')
# read original data
original = annotator.read_original_data(
image_files, original_path, ".jpg")
assert isinstance(original, bob.bio.video.FrameContainer)
assert len(original) == 1
assert original[0][0] == os.path.basename(
image_files.make_path(original_path, ".jpg"))
# annotate data
annot = annotator(original)
assert isinstance(annot, collections.OrderedDict), annot
_assert_bob_ip_facedetect(annot['testimage.jpg'])
def test_wrapper_normalize():
original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
video_object = bob.bio.video.database.VideoBioFile(
client_id=1, file_id=1, path="data/testvideo")
# here I am using 3 frames to test normalize but in real applications this
# should not be done.
frame_selector = bob.bio.video.FrameSelector(
max_number_of_frames=3, selection_style="spread")
annotator = bob.bio.video.annotator.Wrapper(
'flandmark', frame_selector=frame_selector, normalize=True)
video = annotator.read_original_data(video_object, original_path, ".avi")
assert isinstance(video, bob.bio.video.FrameContainer)
annot = annotator(video)
# check if annotations are ordered by frame number
assert list(annot.keys()) == sorted(annot.keys(), key=int), annot
def test_failsafe_video():
original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
video_object = bob.bio.video.database.VideoBioFile(
client_id=1, file_id=1, path="data/testvideo")
# here I am using 3 frames to test normalize but in real applications this
# should not be done.
frame_selector = bob.bio.video.FrameSelector(
max_number_of_frames=3, selection_style="spread")
annotator = bob.bio.video.annotator.FailSafeVideo(
[FailSucessAnnotator(), 'facedetect'], frame_selector=frame_selector)
video = annotator.read_original_data(video_object, original_path, ".avi")
assert isinstance(video, bob.bio.video.FrameContainer)
annot = annotator(video)
# check if annotations are ordered by frame number
assert list(annot.keys()) == sorted(annot.keys(), key=int), annot
# check if the failsuccess annotator was used for all frames
for _, annotations in annot.items():
assert 'topleft' in annotations, annot
assert annotations['topleft'] == (0, 0), annot
assert annotations['bottomright'] == (64, 64), annot
.. _bob.bio.video.annotators:
==========================
Annotating Video Databases
==========================
Please see :ref:`bob.bio.base.annotations` and :ref:`bob.bio.face.annotators`
first before reading this page.
This package provides two wrappers that can be used to annotate video
databases: :any:`bob.bio.video.annotator.Wrapper` and
:any:`bob.bio.video.annotator.FailSafeVideo` which enable you to use image
based annotators on video sequences.
......@@ -16,6 +16,15 @@ Summary
bob.bio.video.extractor.Wrapper
bob.bio.video.algorithm.Wrapper
Annotators
~~~~~~~~~~
.. autosummary::
bob.bio.video.annotator.Base
bob.bio.video.annotator.Wrapper
bob.bio.video.annotator.FailSafeVideo
Databases
~~~~~~~~~
......@@ -24,12 +33,13 @@ Databases
bob.bio.video.database.MobioBioDatabase
bob.bio.video.database.YoutubeBioDatabase
Details
-------
.. automodule:: bob.bio.video
.. automodule:: bob.bio.video.annotator
.. automodule:: bob.bio.video.preprocessor
.. automodule:: bob.bio.video.extractor
......
......@@ -24,6 +24,7 @@ Users Guide
:maxdepth: 2
implementation
annotators
Reference Manual
================
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment