Commit 7bd45d53 authored by Manuel Günther's avatar Manuel Günther

Merge branch 'annotator' into 'master'

Add video annotators

See merge request !28
parents dc7589fe c23e4027
Pipeline #18370 passed with stages
in 62 minutes and 22 seconds
......@@ -4,6 +4,7 @@ from . import extractor
from . import algorithm
from . import database
from . import test
from . import annotator
def get_config():
"""Returns a string containing the configuration information.
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import six
import bob.bio.base
import bob.io.base
......@@ -36,7 +37,7 @@ class Wrapper (bob.bio.base.algorithm.Algorithm):
compressed_io = False
):
# load algorithm configuration
if isinstance(algorithm, str):
if isinstance(algorithm, six.string_types):
self.algorithm = bob.bio.base.load_resource(algorithm, "algorithm")
elif isinstance(algorithm, bob.bio.base.algorithm.Algorithm):
self.algorithm = algorithm
......
import bob.bio.base
from .. import utils
class Base(bob.bio.base.annotator.Annotator):
"""The base class for video annotators.
Parameters
----------
frame_selector : :any:`bob.bio.video.FrameSelector`
A frame selector class to define, which frames of the video to use.
read_original_data : callable
A function with the signature of
``data = read_original_data(biofile, directory, extension)``
that will be used to load the data from biofiles. By default the
``frame_selector`` is used to load the data.
"""
def __init__(self, frame_selector=utils.FrameSelector(selection_style='all'),
read_original_data=None, **kwargs):
def _read_video_data(biofile, directory, extension):
"""Read video data using the frame_selector of this object"""
return biofile.load(directory, extension, frame_selector)
if read_original_data is None:
read_original_data = _read_video_data
super(Base, self).__init__(read_original_data=read_original_data, **kwargs)
@staticmethod
def frame_ids_and_frames(frames):
"""Normalizes the frames parameter into a numpy array and a list of
indices.
Parameters
----------
frames : :any:`bob.bio.video.FrameContainer` or :any:`numpy.array`
The frames of the video file.
Returns
-------
frame_ids : list
A list of strings that represent the frame ids.
frames : :any:`numpy.array`
The frames of the video file as a numpy array.
"""
if isinstance(frames, utils.FrameContainer):
frame_ids = [fid for fid, _, _ in frames]
frames = frames.as_array()
else: # frames is already a numpy.array
frame_ids = [str(i) for i in range(len(frames))]
return frame_ids, frames
def annotate(self, frames, **kwargs):
"""Annotates videos.
Parameters
----------
frames : :any:`bob.bio.video.FrameContainer` or :any:`numpy.array`
The frames of the video file.
**kwargs
Extra arguments that annotators may need.
Returns
-------
collections.OrderedDict
A dictionary where its key is the frame id as a string and its value
is a dictionary that are the annotations for that frame.
.. note::
You can use the :any:`Base.frame_ids_and_frames` functions to normalize
the input in your implementation.
"""
raise NotImplementedError()
from . import Base
import bob.bio.face
import collections
import logging
import six
logger = logging.getLogger(__name__)
class FailSafeVideo(Base):
"""A fail-safe video annotator.
It tries several annotators in order and tries the next one if the previous
one fails. However, the difference between this annotator and
:any:`bob.bio.base.annotator.FailSafe` is that this one tries to use
annotations from older frames (if valid) before trying the next annotator.
.. warning::
You must be careful in using this annotator since different annotators
could have different results. For example the bounding box of one
annotator be totally different from another annotator.
Parameters
----------
annotators : :any:`list`
A list of annotators to try.
max_age : int
The maximum number of frames that an annotation is valid for next frames.
This value should be positive. If you want to set max_age to infinite,
then you can use the :any:`bob.bio.video.annotator.Wrapper` instead.
validator : callable
A function that takes the annotations of a frame and validates it.
Please see :any:`Base` for more accepted parameters.
"""
def __init__(self, annotators, max_age=15,
validator=bob.bio.face.annotator.min_face_size_validator,
**kwargs):
super(FailSafeVideo, self).__init__(**kwargs)
assert max_age > 0, "max_age: `{}' cannot be less than 1".format(max_age)
self.annotators = []
for annotator in annotators:
if isinstance(annotator, six.string_types):
annotator = bob.bio.base.load_resource(annotator, 'annotator')
self.annotators.append(annotator)
self.max_age = max_age
self.validator = validator
def annotate(self, frames, **kwargs):
"""See :any:`Base.annotate`
"""
frame_ids, frames = self.frame_ids_and_frames(frames)
annotations = collections.OrderedDict()
current = None
age = 0
for i, frame in zip(frame_ids, frames):
for annotator in self.annotators:
annot = annotator.annotate(frame, **kwargs)
if annot and self.validator(annot):
current = annot
age = 0
break
elif age < self.max_age:
age += 1
break
else: # no detections and age is larger than maximum allowed
current = None
if current is not annot:
logger.debug("Annotator `%s' failed.", annotator)
annotations[i] = current
return annotations
import six
import collections
import bob.bio.base
import bob.bio.face
from . import Base, normalize_annotations
class Wrapper(Base):
"""Annotates video files using the provided image annotator.
See the documentation of :any:`Base` too.
Parameters
----------
annotator : :any:`bob.bio.base.annotator.Annotator` or str
The image annotator to be used. The annotator could also be the name of a
bob.bio.annotator resource which will be loaded.
max_age : int
see :any:`normalize_annotations`.
normalize : bool
If True, it will normalize annotations using :any:`normalize_annotations`
validator : object
See :any:`normalize_annotations` and
:any:`bob.bio.face.annotator.min_face_size_validator` for one example.
Please see :any:`Base` for more accepted parameters.
.. warning::
You should only set ``normalize`` to True only if you are annotating
**all** frames of the video file.
"""
def __init__(self,
annotator,
normalize=False,
validator=bob.bio.face.annotator.min_face_size_validator,
max_age=-1,
**kwargs
):
super(Wrapper, self).__init__(**kwargs)
self.annotator = annotator
self.normalize = normalize
self.validator = validator
self.max_age = max_age
# load annotator configuration
if isinstance(annotator, six.string_types):
self.annotator = bob.bio.base.load_resource(annotator, "annotator")
def annotate(self, frames, **kwargs):
"""See :any:`Base.annotate`
"""
frame_ids, frames = self.frame_ids_and_frames(frames)
annotations = collections.OrderedDict()
for i, frame in zip(frame_ids, frames):
annotations[i] = self.annotator(frame, **kwargs)
if self.normalize:
annotations = collections.OrderedDict(normalize_annotations(
annotations, self.validator, self.max_age))
return annotations
def normalize_annotations(annotations, validator, max_age=-1):
"""Normalizes the annotations of one video sequence. It fills the
annotations for frames from previous ones if the annotation for the current
frame is not valid.
Parameters
----------
annotations : collections.OrderedDict
A dict of dict where the keys to the first dict are frame indices as
strings (starting from 0). The inside dicts contain annotations for that
frame. The dictionary needs to be an ordered dict in order for this to
work.
validator : callable
Takes a dict (annotations) and returns True if the annotations are valid.
This can be a check based on minimal face size for example: see
:any:`bob.bio.face.annotator.min_face_size_validator`.
max_age : :obj:`int`, optional
An integer indicating for a how many frames a detected face is valid if
no detection occurs after such frame. A value of -1 == forever
Yields
------
str
The index of frame.
dict
The corrected annotations of the frame.
"""
# the annotations for the current frame
current = None
age = 0
for k, annot in annotations.items():
if validator(annot):
current = annot
age = 0
elif max_age < 0 or age < max_age:
age += 1
else: # no detections and age is larger than maximum allowed
current = None
yield k, current
# these imports should be here to avoid circular dependencies
from .Base import Base
from .Wrapper import Wrapper
from .FailSafeVideo import FailSafeVideo
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is
shortened.
Parameters
----------
*args
An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
__appropriate__(
Base,
Wrapper,
FailSafeVideo,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -4,6 +4,7 @@
import bob.bio.base
import bob.io.base
import os
import six
from .. import utils
......@@ -36,7 +37,7 @@ class Wrapper (bob.bio.base.extractor.Extractor):
compressed_io = False
):
# load extractor configuration
if isinstance(extractor, str):
if isinstance(extractor, six.string_types):
self.extractor = bob.bio.base.load_resource(extractor, "extractor")
elif isinstance(extractor, bob.bio.base.extractor.Extractor):
self.extractor = extractor
......
......@@ -3,6 +3,7 @@
import bob.bio.base
import bob.io.base
import six
from .. import utils
......@@ -65,7 +66,7 @@ class Wrapper(bob.bio.base.preprocessor.Preprocessor):
read_original_data = _read_video_data
# load preprocessor configuration
if isinstance(preprocessor, str):
if isinstance(preprocessor, six.string_types):
self.preprocessor = bob.bio.base.load_resource(preprocessor, "preprocessor")
elif isinstance(preprocessor, bob.bio.base.preprocessor.Preprocessor):
self.preprocessor = preprocessor
......@@ -181,4 +182,4 @@ class Wrapper(bob.bio.base.preprocessor.Preprocessor):
if self.compressed_io:
return utils.save_compressed(frames, filename, self.preprocessor.write_data)
else:
frames.save(bob.io.base.HDF5File(filename, 'w'), self.preprocessor.write_data)
\ No newline at end of file
frames.save(bob.io.base.HDF5File(filename, 'w'), self.preprocessor.write_data)
import os
import collections
import bob.io.base
import bob.io.image
import bob.io.video
import bob.bio.video
import pkg_resources
from bob.bio.video.test.dummy.database import DummyBioFile
from bob.bio.face.test.test_annotators import _assert_bob_ip_facedetect
class FailSucessAnnotator(bob.bio.base.annotator.Annotator):
"""An annotator that fails for every second time it is called."""
def __init__(self, **kwargs):
super(FailSucessAnnotator, self).__init__(**kwargs)
self.failed_last_time = True
def annotate(self, image, **kwargs):
if not self.failed_last_time:
self.failed_last_time = True
return None
else:
self.failed_last_time = False
return {
'topleft': (0, 0),
'bottomright': (64, 64)
}
def test_wrapper():
original_path = pkg_resources.resource_filename("bob.bio.face.test", "")
image_files = DummyBioFile(client_id=1, file_id=1, path="data/testimage")
# video preprocessor using a face crop preprocessor
annotator = bob.bio.video.annotator.Wrapper('facedetect')
# read original data
original = annotator.read_original_data(
image_files, original_path, ".jpg")
assert isinstance(original, bob.bio.video.FrameContainer)
assert len(original) == 1
assert original[0][0] == os.path.basename(
image_files.make_path(original_path, ".jpg"))
# annotate data
annot = annotator(original)
assert isinstance(annot, collections.OrderedDict), annot
_assert_bob_ip_facedetect(annot['testimage.jpg'])
def test_wrapper_normalize():
original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
video_object = bob.bio.video.database.VideoBioFile(
client_id=1, file_id=1, path="data/testvideo")
# here I am using 3 frames to test normalize but in real applications this
# should not be done.
frame_selector = bob.bio.video.FrameSelector(
max_number_of_frames=3, selection_style="spread")
annotator = bob.bio.video.annotator.Wrapper(
'flandmark', frame_selector=frame_selector, normalize=True)
video = annotator.read_original_data(video_object, original_path, ".avi")
assert isinstance(video, bob.bio.video.FrameContainer)
annot = annotator(video)
# check if annotations are ordered by frame number
assert list(annot.keys()) == sorted(annot.keys(), key=int), annot
def test_failsafe_video():
original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
video_object = bob.bio.video.database.VideoBioFile(
client_id=1, file_id=1, path="data/testvideo")
# here I am using 3 frames to test normalize but in real applications this
# should not be done.
frame_selector = bob.bio.video.FrameSelector(
max_number_of_frames=3, selection_style="spread")
annotator = bob.bio.video.annotator.FailSafeVideo(
[FailSucessAnnotator(), 'facedetect'], frame_selector=frame_selector)
video = annotator.read_original_data(video_object, original_path, ".avi")
assert isinstance(video, bob.bio.video.FrameContainer)
annot = annotator(video)
# check if annotations are ordered by frame number
assert list(annot.keys()) == sorted(annot.keys(), key=int), annot
# check if the failsuccess annotator was used for all frames
for _, annotations in annot.items():
assert 'topleft' in annotations, annot
assert annotations['topleft'] == (0, 0), annot
assert annotations['bottomright'] == (64, 64), annot
......@@ -7,6 +7,7 @@ import bob.io.image
import bob.io.video
import numpy
import os
import six
import logging
logger = logging.getLogger("bob.bio.video")
......@@ -49,7 +50,7 @@ class FrameSelector:
When giving ``str`` or ``[str]`` data, the given ``load_function`` is used to read the data from file.
"""
# if given a string, first load the video
if isinstance(data, str):
if isinstance(data, six.string_types):
logger.debug("Loading video file '%s'", data)
data = load_function(data)
......
.. _bob.bio.video.annotators:
==========================
Annotating Video Databases
==========================
Please see :ref:`bob.bio.base.annotations` and :ref:`bob.bio.face.annotators`
first before reading this page.
This package provides two wrappers that can be used to annotate video
databases: :any:`bob.bio.video.annotator.Wrapper` and
:any:`bob.bio.video.annotator.FailSafeVideo` which enable you to use image
based annotators on video sequences.
......@@ -16,6 +16,15 @@ Summary
bob.bio.video.extractor.Wrapper
bob.bio.video.algorithm.Wrapper
Annotators
~~~~~~~~~~
.. autosummary::
bob.bio.video.annotator.Base
bob.bio.video.annotator.Wrapper
bob.bio.video.annotator.FailSafeVideo
Databases
~~~~~~~~~
......@@ -24,12 +33,13 @@ Databases
bob.bio.video.database.MobioBioDatabase
bob.bio.video.database.YoutubeBioDatabase
Details
-------
.. automodule:: bob.bio.video
.. automodule:: bob.bio.video.annotator
.. automodule:: bob.bio.video.preprocessor
.. automodule:: bob.bio.video.extractor
......
......@@ -24,6 +24,7 @@ Users Guide
:maxdepth: 2
implementation
annotators
Reference Manual
================
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment