Commit 80703b8f authored by David GEISSBUHLER's avatar David GEISSBUHLER
Browse files

Merge branch 'opencv_bob_conv' into 'master'

Opencv bob conv

See merge request !10
parents c579c0dd 27dfc25e
Pipeline #47319 failed with stages
in 5 minutes and 28 seconds
......@@ -15,7 +15,7 @@
=============
New package
Bob IP Stereo
=============
This package is part of the signal-processing and machine learning toolbox Bob_.
......
......@@ -10,6 +10,7 @@ import bob.ip.color
#==============================================================================
def load_camera_config(filepath, camera_name=None):
""" Load camera config from a JSON file."""
camera_list = {}
with open(filepath, 'r') as data_config_file:
data_config = json.load(data_config_file)
......@@ -31,6 +32,24 @@ def load_camera_config(filepath, camera_name=None):
#==============================================================================
class Camera:
""" This class represents the camera view in 3D space together with its
projection matrix and lens distortion. In addition, 4 marker points
can be added to perform 2D perspective transformations.
Attributes
----------
camera_matrix : :obj:`numpy.ndarray`
Camera projection matrix, according to OpenCV's standards.
dist_coeffs : :obj:`numpy.ndarray`
Distortion coefficeints, according to OpenCV's standards.
R : :obj:`numpy.ndarray`
Rotation matrix relative to left stereo camera.
T : :obj:`numpy.ndarray`
Translation vector relative to left stereo camera.
markers : :obj:`numpy.ndarray`
Four points in 2D image coordinates that can be maapped
to another set of four points in a different camera view.
"""
def __init__(self, camera_matrix, dist_coeffs, R, T, markers):
self.camera_matrix = camera_matrix
......@@ -40,8 +59,26 @@ class Camera:
self.markers = markers
#==============================================================================
class CameraPair:
""" This class represents the camera view in 3D space together with its
projection matrix and lens distortion. In addition, 4 marker points
can be added to perform 2D perspective transformations.
Attributes
----------
camera_matrix : :obj:`numpy.ndarray`
Camera projection matrix, according to OpenCV's standards.
dist_coeffs : :obj:`numpy.ndarray`
Distortion coefficeints, according to OpenCV's standards.
R : :obj:`numpy.ndarray`
Rotation matrix relative to left stereo camera.
T : :obj:`numpy.ndarray`
Translation vector relative to left stereo camera.
markers : :obj:`numpy.ndarray`
Four points in 2D image coordinates that can be maapped
to another set of four points in a different camera view.
"""
def __init__(self, camera_left, camera_right):
self.camera_matrix_l = camera_left.camera_matrix
......
{
"algorythm": 0,
"algorithm": 0,
"min_disparity": 128,
"num_disparities": 96,
"block_size": 15,
......
import json
class StereoParameters:
""" This class stores parameters for the OpenCV's
stereo semi-global block matching algorithm (SGBM).
Please refer to OpenCV's documentation for more information.
Attributes
----------
min_disparity : int
Minimum possible disparity value.
num_disparities : int
Maximum disparity minus minimum disparity, this parameter must be divisible by 16.
blockSize : int
Matched block size, in the 3-11 range.
normalise : bool
Normalise left and right images.
downscale : bool
Downscale left and right images.
erode : bool
Erode disparity holes borders before inpaint.
inpaint : bool
Inpaint holes in the disparity map.
"""
STEREO_CV_SGBM = 0
def __init__( self,
algorythm = STEREO_CV_SGBM,
algorithm = STEREO_CV_SGBM,
min_disparity = 128,
num_disparities = 96,
block_size = 15,
......@@ -42,10 +63,12 @@ class StereoParameters:
self.inpaint = inpaint
def save(self, filepath):
""" Save parameters to a JSON file."""
with open(filepath, 'w') as f:
json.dump(self, f, default=lambda x: x.__dict__, indent=4)
def load(self, filepath):
""" Load parameters from a JSON file."""
with open(filepath) as f:
data = json.load(f)
for key in self.__dict__:
......
......@@ -4,8 +4,6 @@
import numpy as np
import cv2 as cv
import datetime
import bob.io.base
import bob.ip.color
......@@ -15,11 +13,35 @@ from .utils import convert_to_uint8
from ._library import remap, project_map, project_bounding_box, project_image_points
#==============================================================================
# perform block matching and return 3d image
def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()):
#a = datetime.datetime.now()
""" This function performs stereo reconstruction on a pair of images.
The reconstruction process begins by converting the input image pair to
grayscale and then rectifying them using the :class:`~bob.ip.stereo.CameraPair`
object, if requested a 2x downscaling is also performed. Image disparity is then
computed accordingly to the algorithm and parameters described in the
:class:`~bob.ip.stereo.StereoParameter` object, if requested in these options voids
in the resulting disparity map are filled using an in-painting algorithm.
Finally, the disparity map is converted to a 3D map using geometric
information from the :class:`~bob.ip.stereo.CameraPair` object.
Parameters
----------
img_l : :obj:`numpy.ndarray`
Left image.
img_r : :obj:`numpy.ndarray`
Right image.
camera_pair : :obj:`~bob.ip.stereo.CameraPair`
Camera pair object containing geometric information of the cameras.
stereo_parameters: :obj:`~bob.ip.stereo.StereoParameter`
Parameters for the stereo reconstruction algorithm.
Returns
-------
:obj:`numpy.ndarray`
The resulting 3D reconstructed scene.
"""
# convert to gray
......@@ -105,12 +127,48 @@ def stereo_match(img_l, img_r, camera_pair, stereo_parameters=StereoParameters()
img3d = np.rollaxis(img3d, 2)
#print('stereo -> ', (datetime.datetime.now() - a).total_seconds())
return img3d
#==============================================================================
def reproject_image(img, img3d, camera, camera_pair, bounding_box=None, image_points=None):
""" This function projects a camera view at any point in 3D space on the 3D point
cloud obtained by stereo reconstruction and re-project back on the left camera
view, thus allowing for instance precise alignment of an RGB camera on the
3D stereo image.
This process works in two steps. In a first stage the 3D point cloud is projected
on the camera view to be re-mapped, taking into account the distortion of this
particular camera, yielding a mapping from the left rectified 2D coordinates to the
camera under consideration 2D view. In a second stage this mapping is inverted and the
image is re-mapped to the left rectified view. The resolution of the source image is
automatically rescaled to match the resolution of the stereo view. In addition, it is
possible to transform a bounding box object and an arbitrary number of image points
(for instance an object bounding box and landmarks on the original 2D image).
Parameters
----------
img : :obj:`numpy.ndarray`
Image to be re-projected (int8, int16 and float64 are supported).
img3d : :obj:`numpy.ndarray`
3D image (point cloud) of the scene.
camera : :obj:`~bob.ip.stereo.Camera`
Camera object containing geometric information of the camera to be re-projected.
camera_pair : :obj:`~bob.ip.stereo.CameraPair`
Camera pair object containing geometric information of the stereo cameras.
bounding_box : :obj:`numpy.ndarray`
2x2 numpy array [[top, bottom], [left, right]] bounding box (int64), the array is
modified by the functions.
image_point : :obj:`numpy.ndarray`
Nx2 numpy array containing coordinates of the image points, the array is modified by
the function.
Returns
-------
:obj:`numpy.ndarray`
The resulting reprojected image.
"""
# shape
......
......@@ -7,12 +7,14 @@ Test Units
# ==============================================================================
from bob.ip.stereo.stereo import stereo_match, reproject_image, StereoParameters
from bob.ip.stereo.camera import load_camera_config, CameraPair
from bob.ip.stereo.utils import convert_bob_to_cv, convert_cv_to_bob
from bob.io.base import load
from bob.io.image import load
from pkg_resources import resource_filename
import numpy as np
import cv2
# ==============================================================================
......@@ -27,6 +29,21 @@ def resource_path(relative_path, package="bob.ip.stereo"):
return resource_filename(package, relative_path)
# ==============================================================================
# test utils
def test_image_format_conversion():
"""Unit tests for :func:`~bob.io.stream.utils.convert_bob_to_cv` and :func:`~bob.io.stream.utils.convert_cv_to_bob`."""
bob_im = load(resource_path("test/data/color.png"))
ocv_im = cv2.imread(resource_path("test/data/color.png"))
assert np.array_equal(ocv_im, convert_bob_to_cv(bob_im))
assert np.array_equal(bob_im, convert_cv_to_bob(ocv_im))
# ==============================================================================
# Image distances functions
def sum_of_squared_absolute_difference(array1, array2):
"""Sum the squared absolute difference between the input arrays.
......@@ -59,6 +76,9 @@ def canberra_distance(array1, array2):
)
# ==============================================================================
# Image distance aggregate function
def is_close_enough(image1, image2):
"""Checks if the 2 inputs are close enough to pass the test, using different metrics.
......
import numpy as np
import cv2 as cv
# convert from OpenCV's <h, w, BGR> to bob's <RGB, h, w> and vice versa
def convert_bob_to_cv(img):
assert len(img.shape) == 3
assert img.shape[0] == 3
img = np.rollaxis(img, 2)
"""Convert image from Bob's <RGB, h, w> format to OpenCV's <h, w, BGR> format.
Parameters
----------
img : :obj:`numpy.ndarray`
Image in Bob's format <RGB, h, w>.
Returns
-------
:obj:`numpy.ndarray`
Image in OpenCV's format <h, w, BGR>.
Raises
------
ValueError
If the input image has the wrong number of dimensions. (Must be 3).
ValueError
If the input image does not have 3 channels in first dimension.
"""
if len(img.shape) != 3:
raise ValueError("Expected image to be 3D object, but got shape " + str(img.shape))
if img.shape[0] != 3:
raise ValueError("Expected image to have 3 color channels, but got " + str(img.shape[0]) + " channels.")
img = np.moveaxis(img, 0, 2)
img = np.flip(img, axis=2)
return img
def convert_cv_to_bob(img):
assert len(img.shape) == 3
assert img.shape[2] == 3
img = np.rollaxis(img, 2)
"""Convert image from OpenCV's <h, w, BGR> format to Bob's <RGB, h, w> format.
Parameters
----------
img : :obj:`numpy.ndarray`
Image in OpenCV's <h, w, BGR> format.
Returns
-------
:obj:`numpy.ndarray`
Image in Bob's <RGB, h, w> format.
Raises
------
ValueError
If the input image has the wrong number of dimension. (Must be 3).
ValueError
If the input image does not have 3 channels in the last dimension.
"""
if len(img.shape) != 3:
raise ValueError("Expected image to have 3 dimension, but got shape " + str(img.shape))
if img.shape[2] != 3:
raise ValueError("Expected image to have 3 color channels, but got " + str(img.shape[2]) + " channels.")
img = np.moveaxis(img, 2, 0)
img = np.flip(img, axis=0)
return img
def convert_to_uint8(img, normalize=False):
if img.dtype == np.uint16:
if normalize:
......
......@@ -3,14 +3,13 @@
.. _bob.ip.stereo:
=============
New package
Bob IP Stereo
=============
.. todo ::
Write here a small (1 paragraph) introduction explaining this project. See
other projects for examples.
This package provides a wrapper to OpenCV's stereo reconstruction algorithm through
an user friendly interface. In addition, algorithms allowing re-projection (or re-mapping)
of an arbitrary camera viewport image into another is provided, allowing seemless
use of images, or videos, captured in multi-camera machine vision setups.
Users Guide
===========
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment