Commit 08cac861 authored by Vincent POLLET's avatar Vincent POLLET
Browse files

Merge branch 'master' into 'idiap_opencv'

# Conflicts:
#   bob/ip/stereo/stereo.py
#   doc/index.rst
parents 8c02b77a 1602c2ec
Pipeline #47381 failed with stage
in 11 minutes and 42 seconds
......@@ -15,7 +15,7 @@
=============
New package
Bob IP Stereo
=============
This package is part of the signal-processing and machine learning toolbox Bob_.
......
......@@ -9,6 +9,7 @@ import bob.ip.color
#==============================================================================
def load_camera_config(filepath, camera_name=None):
""" Load camera config from a JSON file."""
camera_list = {}
with open(filepath, 'r') as data_config_file:
data_config = json.load(data_config_file)
......@@ -30,6 +31,24 @@ def load_camera_config(filepath, camera_name=None):
#==============================================================================
class Camera:
""" This class represents the camera view in 3D space together with its
projection matrix and lens distortion. In addition, 4 marker points
can be added to perform 2D perspective transformations.
Attributes
----------
camera_matrix : :obj:`numpy.ndarray`
Camera projection matrix, according to OpenCV's standards.
dist_coeffs : :obj:`numpy.ndarray`
Distortion coefficeints, according to OpenCV's standards.
R : :obj:`numpy.ndarray`
Rotation matrix relative to left stereo camera.
T : :obj:`numpy.ndarray`
Translation vector relative to left stereo camera.
markers : :obj:`numpy.ndarray`
Four points in 2D image coordinates that can be maapped
to another set of four points in a different camera view.
"""
def __init__(self, camera_matrix, dist_coeffs, R, T, markers):
self.camera_matrix = camera_matrix
......@@ -39,8 +58,26 @@ class Camera:
self.markers = markers
#==============================================================================
class CameraPair:
""" This class represents the camera view in 3D space together with its
projection matrix and lens distortion. In addition, 4 marker points
can be added to perform 2D perspective transformations.
Attributes
----------
camera_matrix : :obj:`numpy.ndarray`
Camera projection matrix, according to OpenCV's standards.
dist_coeffs : :obj:`numpy.ndarray`
Distortion coefficeints, according to OpenCV's standards.
R : :obj:`numpy.ndarray`
Rotation matrix relative to left stereo camera.
T : :obj:`numpy.ndarray`
Translation vector relative to left stereo camera.
markers : :obj:`numpy.ndarray`
Four points in 2D image coordinates that can be maapped
to another set of four points in a different camera view.
"""
def __init__(self, camera_left, camera_right):
self.camera_matrix_l = camera_left.camera_matrix
......
{
"algorythm": 0,
"algorithm": 0,
"min_disparity": 128,
"num_disparities": 96,
"block_size": 15,
......
import json
class StereoParameters:
""" This class stores parameters for the OpenCV's
stereo semi-global block matching algorithm (SGBM).
Please refer to OpenCV's documentation for more information.
Attributes
----------
min_disparity : int
Minimum possible disparity value.
num_disparities : int
Maximum disparity minus minimum disparity, this parameter must be divisible by 16.
blockSize : int
Matched block size, in the 3-11 range.
normalise : bool
Normalise left and right images.
downscale : bool
Downscale left and right images.
erode : bool
Erode disparity holes borders before inpaint.
inpaint : bool
Inpaint holes in the disparity map.
"""
STEREO_CV_SGBM = 0
def __init__( self,
algorythm = STEREO_CV_SGBM,
algorithm = STEREO_CV_SGBM,
min_disparity = 128,
num_disparities = 96,
block_size = 15,
......@@ -20,7 +41,7 @@ class StereoParameters:
erode = False,
inpaint = False):
self.algorythm = algorythm
self.algorithm = algorithm
# OPENCV_SGBM
self.min_disparity = min_disparity
......@@ -42,10 +63,12 @@ class StereoParameters:
self.inpaint = inpaint
def save(self, filepath):
""" Save parameters to a JSON file."""
with open(filepath, 'w') as f:
json.dump(self, f, default=lambda x: x.__dict__, indent=4)
def load(self, filepath):
""" Load parameters from a JSON file."""
with open(filepath) as f:
data = json.load(f)
for key in self.__dict__:
......
......@@ -15,11 +15,35 @@ from .utils import convert_to_uint8
from ._library import remap, project_map, project_bounding_box, project_image_points
#==============================================================================
# perform block matching and return 3d image
def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()):
#a = datetime.datetime.now()
""" This function performs stereo reconstruction on a pair of images.
The reconstruction process begins by converting the input image pair to
grayscale and then rectifying them using the :class:`~bob.ip.stereo.CameraPair`
object, if requested a 2x downscaling is also performed. Image disparity is then
computed accordingly to the algorithm and parameters described in the
:class:`~bob.ip.stereo.StereoParameter` object, if requested in these options voids
in the resulting disparity map are filled using an in-painting algorithm.
Finally, the disparity map is converted to a 3D map using geometric
information from the :class:`~bob.ip.stereo.CameraPair` object.
Parameters
----------
img_l : :obj:`numpy.ndarray`
Left image.
img_r : :obj:`numpy.ndarray`
Right image.
camera_pair : :obj:`~bob.ip.stereo.CameraPair`
Camera pair object containing geometric information of the cameras.
stereo_parameters: :obj:`~bob.ip.stereo.StereoParameter`
Parameters for the stereo reconstruction algorithm.
Returns
-------
:obj:`numpy.ndarray`
The resulting 3D reconstructed scene.
"""
# convert to gray
......@@ -54,7 +78,7 @@ def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()
#block match
if stereo_parameters.algorythm == stereo_parameters.STEREO_CV_SGBM:
if stereo_parameters.algorithm == stereo_parameters.STEREO_CV_SGBM:
stereo = cv.StereoSGBM_create(minDisparity = stereo_parameters.min_disparity,
numDisparities = stereo_parameters.num_disparities,
......@@ -74,7 +98,7 @@ def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()
else:
raise Exception('Unknown stereo algorythm')
raise Exception('Unknown stereo algorithm')
# inpaint depth map holes
......@@ -105,12 +129,48 @@ def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()
img3d = np.rollaxis(img3d, 2)
#print('stereo -> ', (datetime.datetime.now() - a).total_seconds())
return img3d
#==============================================================================
def reproject_image(img, img3d, camera, camera_pair, bounding_box=None, image_points=None):
""" This function projects a camera view at any point in 3D space on the 3D point
cloud obtained by stereo reconstruction and re-project back on the left camera
view, thus allowing for instance precise alignment of an RGB camera on the
3D stereo image.
This process works in two steps. In a first stage the 3D point cloud is projected
on the camera view to be re-mapped, taking into account the distortion of this
particular camera, yielding a mapping from the left rectified 2D coordinates to the
camera under consideration 2D view. In a second stage this mapping is inverted and the
image is re-mapped to the left rectified view. The resolution of the source image is
automatically rescaled to match the resolution of the stereo view. In addition, it is
possible to transform a bounding box object and an arbitrary number of image points
(for instance an object bounding box and landmarks on the original 2D image).
Parameters
----------
img : :obj:`numpy.ndarray`
Image to be re-projected (int8, int16 and float64 are supported).
img3d : :obj:`numpy.ndarray`
3D image (point cloud) of the scene.
camera : :obj:`~bob.ip.stereo.Camera`
Camera object containing geometric information of the camera to be re-projected.
camera_pair : :obj:`~bob.ip.stereo.CameraPair`
Camera pair object containing geometric information of the stereo cameras.
bounding_box : :obj:`numpy.ndarray`
2x2 numpy array [[top, bottom], [left, right]] bounding box (int64), the array is
modified by the functions.
image_point : :obj:`numpy.ndarray`
Nx2 numpy array containing coordinates of the image points, the array is modified by
the function.
Returns
-------
:obj:`numpy.ndarray`
The resulting reprojected image.
"""
# shape
......
......@@ -13,6 +13,7 @@ from bob.io.image import load
from pkg_resources import resource_filename
import numpy as np
import cv2
# ==============================================================================
......@@ -28,6 +29,9 @@ def resource_path(relative_path, package="bob.ip.stereo"):
return resource_filename(package, relative_path)
# ==============================================================================
# Image distances functions
def sum_of_squared_absolute_difference(array1, array2):
"""Sum the squared absolute difference between the input arrays.
"""
......@@ -59,6 +63,9 @@ def canberra_distance(array1, array2):
)
# ==============================================================================
# Image distance aggregate function
def is_close_enough(image1, image2):
"""Checks if the 2 inputs are close enough to pass the test, using different metrics.
......
import numpy as np
import cv2 as cv
# convert from OpenCV's <h, w, BGR> to bob's <RGB, h, w> and vice versa
def convert_bob_to_cv(img):
assert len(img.shape) == 3
assert img.shape[0] == 3
img = np.rollaxis(img, 2)
img = np.flip(img, axis=2)
return img
def convert_cv_to_bob(img):
assert len(img.shape) == 3
assert img.shape[2] == 3
img = np.rollaxis(img, 2)
img = np.flip(img, axis=0)
return img
def convert_to_uint8(img, normalize=False):
if img.dtype == np.uint16:
if normalize:
......
......@@ -3,14 +3,13 @@
.. _bob.ip.stereo:
=============
New package
Bob IP Stereo
=============
.. todo::
Write here a small (1 paragraph) introduction explaining this project. See
other projects for examples.
This package provides a wrapper to OpenCV's stereo reconstruction algorithm through
an user friendly interface. In addition, algorithms allowing re-projection (or re-mapping)
of an arbitrary camera viewport image into another is provided, allowing seemless
use of images, or videos, captured in multi-camera machine vision setups.
Users Guide
===========
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment