Commit a1c2d24a authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'face-crop-improvements' into 'dask-pipelines'

Face crop improvements

See merge request !71
parents 6ec58c33 38e6503f
Pipeline #44907 failed with stage
in 50 seconds
......@@ -5,9 +5,6 @@ import bob.bio.face.preprocessor # import for documentation
class Base(bob.bio.base.annotator.Annotator):
"""Base class for all face annotators"""
def __init__(self, **kwargs):
super(Base, self).__init__(**kwargs)
def annotate(self, sample, **kwargs):
"""Annotates an image and returns annotations in a dictionary. All
annotator should return at least the ``topleft`` and ``bottomright``
......
......@@ -11,7 +11,6 @@ database = DatabaseConnector(
IJBCBioDatabase(original_directory=ijbc_directory, protocol="1:1"),
annotation_type = "eyes-center",
fixed_positions = None,
allow_scoring_with_all_biometric_references = False
)
#ijbc_covariates = DatabaseConnector(
......
......@@ -12,6 +12,7 @@
from .database import FaceBioFile
from bob.bio.base.database import BioDatabase
from bob.db.base.utils import convert_names_to_highlevel, convert_names_to_lowlevel
class ReplayBioFile(FaceBioFile):
......@@ -64,7 +65,7 @@ class ReplayBioDatabase(BioDatabase):
return names
def groups(self):
return self.convert_names_to_highlevel(
return convert_names_to_highlevel(
self._db.groups(), self.low_level_group_names, self.high_level_group_names)
def annotations(self, file):
......@@ -89,7 +90,7 @@ class ReplayBioDatabase(BioDatabase):
groups = self.check_parameters_for_validity(groups, "group", self.groups(), self.groups())
purposes = self.check_parameters_for_validity(purposes, "purpose", ('enroll', 'probe'), ('enroll', 'probe'))
purposes = list(purposes)
groups = self.convert_names_to_lowlevel(
groups = convert_names_to_lowlevel(
groups, self.low_level_group_names, self.high_level_group_names)
# protocol licit is not defined in the low level API
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
from bob.bio.face.preprocessor import FaceCrop, Scale
from bob.bio.face.preprocessor import FaceDetect, FaceCrop, Scale
from skimage.transform import resize
import numpy as np
def face_crop_solver(
cropped_image_size,
color_channel="rgb",
cropped_positions=None,
color_channel="rgb",
fixed_positions=None,
use_face_detector=False,
dtype=np.uint8
annotator=None,
dtype="uint8",
):
"""
Decide which face cropper to use.
"""
if use_face_detector:
return FaceDetect(
face_cropper="face-crop-eyes", use_flandmark=True
)
# If there's not cropped positions, just resize
if cropped_positions is None:
return Scale(cropped_image_size)
else:
# If there's not cropped positions, just resize
if cropped_positions is None:
return Scale(cropped_image_size)
else:
# Detects the face and crops it without eye detection
return FaceCrop(
cropped_image_size=cropped_image_size,
cropped_positions=cropped_positions,
color_channel=color_channel,
fixed_positions=fixed_positions,
dtype=dtype
)
# Detects the face and crops it without eye detection
return FaceCrop(
cropped_image_size=cropped_image_size,
cropped_positions=cropped_positions,
color_channel=color_channel,
fixed_positions=fixed_positions,
dtype=dtype,
annotator=annotator,
)
import numpy
import bob.io.image
import bob.ip.color
import numpy
from sklearn.base import BaseEstimator
from sklearn.base import TransformerMixin
def change_color_channel(image, color_channel):
if image.ndim == 2:
if color_channel == "rgb":
return bob.ip.color.gray_to_rgb(image)
if color_channel != "gray":
raise ValueError(
"There is no rule to extract a "
+ color_channel
+ " image from a gray level image!"
)
return image
from sklearn.base import TransformerMixin, BaseEstimator
if color_channel == "rgb":
return image
if color_channel == "gray":
return bob.ip.color.rgb_to_gray(image)
if color_channel == "red":
return image[0, :, :]
if color_channel == "green":
return image[1, :, :]
if color_channel == "blue":
return image[2, :, :]
raise ValueError(
"The image channel '%s' is not known or not yet implemented", color_channel
)
class Base(TransformerMixin, BaseEstimator):
"""Performs color space adaptations and data type corrections for the given
image.
image.
**Parameters:**
**Parameters:**
dtype : :py:class:`numpy.dtype` or convertible or ``None``
The data type that the resulting image will have.
dtype : :py:class:`numpy.dtype` or convertible or ``None``
The data type that the resulting image will have.
color_channel : one of ``('gray', 'red', 'gren', 'blue', 'rgb')``
The specific color channel, which should be extracted from the image.
"""
color_channel : one of ``('gray', 'red', 'gren', 'blue', 'rgb')``
The specific color channel, which should be extracted from the image.
"""
def __init__(self, dtype=None, color_channel="gray", **kwargs):
self.channel = color_channel
self.color_channel = color_channel
self.dtype = dtype
@property
def channel(self):
return self.color_channel
def _more_tags(self):
return {"stateless": True, "requires_fit": False}
......@@ -31,85 +63,62 @@ class Base(TransformerMixin, BaseEstimator):
def color_channel(self, image):
"""color_channel(image) -> channel
Returns the channel of the given image, which was selected in the
constructor. Currently, gray, red, green and blue channels are supported.
Returns the channel of the given image, which was selected in the
constructor. Currently, gray, red, green and blue channels are supported.
**Parameters:**
**Parameters:**
image : 2D or 3D :py:class:`numpy.ndarray`
The image to get the specified channel from.
image : 2D or 3D :py:class:`numpy.ndarray`
The image to get the specified channel from.
**Returns:**
**Returns:**
channel : 2D or 3D :py:class:`numpy.ndarray`
The extracted color channel.
"""
channel : 2D or 3D :py:class:`numpy.ndarray`
The extracted color channel.
"""
if image.ndim == 2:
if self.channel == "rgb":
return bob.ip.color.gray_to_rgb(image)
if self.channel != "gray":
raise ValueError(
"There is no rule to extract a "
+ self.channel
+ " image from a gray level image!"
)
return image
if self.channel == "rgb":
return image
if self.channel == "gray":
return bob.ip.color.rgb_to_gray(image)
if self.channel == "red":
return image[0, :, :]
if self.channel == "green":
return image[1, :, :]
if self.channel == "blue":
return image[2, :, :]
raise ValueError(
"The image channel '%s' is not known or not yet implemented", self.channel
)
return change_color_channel(image, self.color_channel)
def data_type(self, image):
"""data_type(image) -> image
Converts the given image into the data type specified in the constructor of
this class. If no data type was specified, or the ``image`` is ``None``, no
conversion is performed.
Converts the given image into the data type specified in the constructor of
this class. If no data type was specified, or the ``image`` is ``None``, no
conversion is performed.
**Parameters:**
**Parameters:**
image : 2D or 3D :py:class:`numpy.ndarray`
The image to convert.
image : 2D or 3D :py:class:`numpy.ndarray`
The image to convert.
**Returns:**
**Returns:**
image : 2D or 3D :py:class:`numpy.ndarray`
The image converted to the desired data type, if any.
"""
image : 2D or 3D :py:class:`numpy.ndarray`
The image converted to the desired data type, if any.
"""
if self.dtype is not None and image is not None:
image = image.astype(self.dtype)
return image
def transform(self, image, annotations=None):
"""__call__(image, annotations = None) -> image
def transform(self, images, annotations=None):
"""Extracts the desired color channel and converts to the desired data type.
Extracts the desired color channel and converts to the desired data type.
**Parameters:**
**Parameters:**
image : 2D or 3D :py:class:`numpy.ndarray`
The image to preprocess.
image : 2D or 3D :py:class:`numpy.ndarray`
The image to preprocess.
annotations : any
Ignored.
annotations : any
Ignored.
**Returns:**
**Returns:**
image : 2D :py:class:`numpy.ndarray`
The image converted converted to the desired color channel and type.
"""
return [self._transform_one_image(img) for img in images]
image : 2D :py:class:`numpy.ndarray`
The image converted converted to the desired color channel and type.
"""
def _transform_one_image(self, image):
assert isinstance(image, numpy.ndarray) and image.ndim in (2, 3)
# convert to grayscale
image = self.color_channel(image)
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Thu May 24 10:41:42 CEST 2012
import bob.ip.base
import numpy
import logging
......@@ -10,101 +5,101 @@ import logging
from .Base import Base
logger = logging.getLogger("bob.bio.face")
from sklearn.utils import check_array
from bob.pipelines.sample import SampleBatch
from bob.bio.base import load_resource
class FaceCrop(Base):
"""Crops the face according to the given annotations.
This class is designed to perform a geometric normalization of the face based
on the eye locations, using :py:class:`bob.ip.base.FaceEyesNorm`. Usually,
when executing the :py:meth:`crop_face` function, the image and the eye
locations have to be specified. There, the given image will be transformed
such that the eye locations will be placed at specific locations in the
resulting image. These locations, as well as the size of the cropped image,
need to be specified in the constructor of this class, as
``cropped_positions`` and ``cropped_image_size``.
Some image databases do not provide eye locations, but rather bounding boxes.
This is not a problem at all.
Simply define the coordinates, where you want your ``cropped_positions`` to
be in the cropped image, by specifying the same keys in the dictionary that
will be given as ``annotations`` to the :py:meth:`crop_face` function.
.. note::
These locations can even be outside of the cropped image boundary, i.e.,
when the crop should be smaller than the annotated bounding boxes.
Sometimes, databases provide pre-cropped faces, where the eyes are located at
(almost) the same position in all images. Usually, the cropping does not
conform with the cropping that you like (i.e., image resolution is wrong, or
too much background information). However, the database does not provide eye
locations (since they are almost identical for all images). In that case, you
can specify the ``fixed_positions`` in the constructor, which will be taken
instead of the ``annotations`` inside the :py:meth:`crop_face` function (in
which case the ``annotations`` are ignored).
Sometimes, the crop of the face is outside of the original image boundaries.
Usually, these pixels will simply be left black, resulting in sharp edges in
the image. However, some feature extractors do not like these sharp edges. In
this case, you can set the ``mask_sigma`` to copy pixels from the valid
border of the image and add random noise (see
:py:func:`bob.ip.base.extrapolate_mask`).
Parameters
----------
cropped_image_size : (int, int)
The resolution of the cropped image, in order (HEIGHT,WIDTH); if not given,
no face cropping will be performed
cropped_positions : dict
The coordinates in the cropped image, where the annotated points should be
put to. This parameter is a dictionary with usually two elements, e.g.,
``{'reye':(RIGHT_EYE_Y, RIGHT_EYE_X) , 'leye':(LEFT_EYE_Y, LEFT_EYE_X)}``.
However, also other parameters, such as ``{'topleft' : ..., 'bottomright' :
...}`` are supported, as long as the ``annotations`` in the `__call__`
function are present.
fixed_positions : dict or None
If specified, ignore the annotations from the database and use these fixed
positions throughout.
mask_sigma : float or None
Fill the area outside of image boundaries with random pixels from the
border, by adding noise to the pixel values. To disable extrapolation, set
this value to ``None``. To disable adding random noise, set it to a
negative value or 0.
mask_neighbors : int
The number of neighbors used during mask extrapolation. See
:py:func:`bob.ip.base.extrapolate_mask` for details.
mask_seed : int or None
The random seed to apply for mask extrapolation.
allow_upside_down_normalized_faces: bool, optional
If ``False`` (default), a ValueError is raised when normalized faces are going to be
upside down compared to input image. This allows you to catch wrong annotations in
your database easily. If you are sure about your input, you can set this flag to
``True``.
.. warning::
When run in parallel, the same random seed will be applied to all
parallel processes. Hence, results of parallel execution will differ
from the results in serial execution.
annotator : :any:`bob.bio.base.annotator.Annotator`
If provided, the annotator will be used if the required annotations are
missing.
kwargs
Remaining keyword parameters passed to the :py:class:`Base` constructor,
such as ``color_channel`` or ``dtype``.
"""
This class is designed to perform a geometric normalization of the face based
on the eye locations, using :py:class:`bob.ip.base.FaceEyesNorm`. Usually,
when executing the :py:meth:`crop_face` function, the image and the eye
locations have to be specified. There, the given image will be transformed
such that the eye locations will be placed at specific locations in the
resulting image. These locations, as well as the size of the cropped image,
need to be specified in the constructor of this class, as
``cropped_positions`` and ``cropped_image_size``.
Some image databases do not provide eye locations, but rather bounding boxes.
This is not a problem at all.
Simply define the coordinates, where you want your ``cropped_positions`` to
be in the cropped image, by specifying the same keys in the dictionary that
will be given as ``annotations`` to the :py:meth:`crop_face` function.
.. note::
These locations can even be outside of the cropped image boundary, i.e.,
when the crop should be smaller than the annotated bounding boxes.
Sometimes, databases provide pre-cropped faces, where the eyes are located at
(almost) the same position in all images. Usually, the cropping does not
conform with the cropping that you like (i.e., image resolution is wrong, or
too much background information). However, the database does not provide eye
locations (since they are almost identical for all images). In that case, you
can specify the ``fixed_positions`` in the constructor, which will be taken
instead of the ``annotations`` inside the :py:meth:`crop_face` function (in
which case the ``annotations`` are ignored).
Sometimes, the crop of the face is outside of the original image boundaries.
Usually, these pixels will simply be left black, resulting in sharp edges in
the image. However, some feature extractors do not like these sharp edges. In
this case, you can set the ``mask_sigma`` to copy pixels from the valid
border of the image and add random noise (see
:py:func:`bob.ip.base.extrapolate_mask`).
Parameters
----------
cropped_image_size : (int, int)
The resolution of the cropped image, in order (HEIGHT,WIDTH); if not given,
no face cropping will be performed
cropped_positions : dict
The coordinates in the cropped image, where the annotated points should be
put to. This parameter is a dictionary with usually two elements, e.g.,
``{'reye':(RIGHT_EYE_Y, RIGHT_EYE_X) , 'leye':(LEFT_EYE_Y, LEFT_EYE_X)}``.
However, also other parameters, such as ``{'topleft' : ..., 'bottomright' :
...}`` are supported, as long as the ``annotations`` in the `__call__`
function are present.
fixed_positions : dict or None
If specified, ignore the annotations from the database and use these fixed
positions throughout.
mask_sigma : float or None
Fill the area outside of image boundaries with random pixels from the
border, by adding noise to the pixel values. To disable extrapolation, set
this value to ``None``. To disable adding random noise, set it to a
negative value or 0.
mask_neighbors : int
The number of neighbors used during mask extrapolation. See
:py:func:`bob.ip.base.extrapolate_mask` for details.
mask_seed : int or None
The random seed to apply for mask extrapolation.
.. warning::
When run in parallel, the same random seed will be applied to all
parallel processes. Hence, results of parallel execution will differ
from the results in serial execution.
allow_upside_down_normalized_faces: bool, optional
If ``False`` (default), a ValueError is raised when normalized faces are going to be
upside down compared to input image. This allows you to catch wrong annotations in
your database easily. If you are sure about your input, you can set this flag to
``True``.
annotator : :any:`bob.bio.base.annotator.Annotator`
If provided, the annotator will be used if the required annotations are
missing.
kwargs
Remaining keyword parameters passed to the :py:class:`Base` constructor,
such as ``color_channel`` or ``dtype``.
"""
def __init__(
self,
......@@ -116,11 +111,36 @@ class FaceCrop(Base):
mask_seed=None,
annotator=None,
allow_upside_down_normalized_faces=False,
**kwargs
**kwargs,
):
Base.__init__(self, **kwargs)
if isinstance(cropped_image_size, int):
cropped_image_size = (cropped_image_size, cropped_image_size)
if isinstance(cropped_positions, str):
face_size = cropped_image_size[0]
if cropped_positions == "eyes-center":
eyes_distance = (face_size + 1) / 2.0
eyes_center = (face_size / 4.0, (face_size - 0.5) / 2.0)
right_eye = (eyes_center[0], eyes_center[1] - eyes_distance / 2)
left_eye = (eyes_center[0], eyes_center[1] + eyes_distance / 2)
cropped_positions = {"reye": right_eye, "leye": left_eye}
elif cropped_positions == "bounding-box":
cropped_positions = {
"topleft": (0, 0),
"bottomright": cropped_image_size,
}
else:
raise ValueError(
f"Got {cropped_positions} as cropped_positions "
"while only eyes and bbox strings are supported."
)
# call base class constructor
self.cropped_image_size = cropped_image_size
self.cropped_positions = cropped_positions
......@@ -142,6 +162,8 @@ class FaceCrop(Base):
self.mask_sigma = mask_sigma
self.mask_neighbors = mask_neighbors
self.mask_seed = mask_seed
if isinstance(annotator, str):
annotator = load_resource(annotator, "annotator")
self.annotator = annotator
self.allow_upside_down_normalized_faces = allow_upside_down_normalized_faces
......@@ -169,28 +191,28 @@ class FaceCrop(Base):
def crop_face(self, image, annotations=None):
"""Crops the face.
Executes the face cropping on the given image and returns the cropped
version of it.
Parameters
----------
image : 2D :py:class:`numpy.ndarray`
The face image to be processed.
annotations : dict or ``None``
The annotations that fit to the given image. ``None`` is only accepted,
when ``fixed_positions`` were specified in the constructor.
Returns
-------
face : 2D :py:class:`numpy.ndarray` (float)
The cropped face.
Raises
------
ValueError
If the annotations is None.
"""
Executes the face cropping on the given image and returns the cropped
version of it.
Parameters
----------
image : 2D :py:class:`numpy.ndarray`
The face image to be processed.
annotations : dict or ``None``
The annotations that fit to the given image. ``None`` is only accepted,
when ``fixed_positions`` were specified in the constructor.
Returns
-------
face : 2D :py:class:`numpy.ndarray` (float)
The cropped face.
Raises
------
ValueError
If the annotations is None.
"""
if self.fixed_positions is not None:
annotations = self.fixed_positions
if annotations is None:
......@@ -282,25 +304,25 @@ class FaceCrop(Base):
def transform(self, X, annotations=None):
"""Aligns the given image according to the given annotations.
First, the desired color channel is extracted from the given image.
Afterward, the face is cropped, according to the given ``annotations`` (or
to ``fixed_positions``, see :py:meth:`crop_face`). Finally, the resulting
face is converted to the desired data type.
Parameters
----------
image : 2D or 3D :py:class:`numpy.ndarray`
The face image to be processed.
annotations : dict or ``None``
The annotations that fit to the given image.
Returns
-------
face : 2D :py:class:`numpy.ndarray`
The cropped face.
"""
def _crop(image, annot):
First, the desired color channel is extracted from the given image.
Afterward, the face is cropped, according to the given ``annotations`` (or
to ``fixed_positions``, see :py:meth:`crop_face`). Finally, the resulting
face is converted to the desired data type.
Parameters
----------
image : 2D or 3D :py:class:`numpy.ndarray`
The face image to be processed.
annotations : dict or ``None``
The annotations that fit to the given image.