Commit 4d14784b authored by Guillaume CLIVAZ's avatar Guillaume CLIVAZ
Browse files

[calibration] Warp calibration script to save markers in config

- used for warp StreamFilter
- rst documentation to create a JSON config and run the script
parent 7ebc672e
Pipeline #52142 failed with stage
in 60 minutes and 4 seconds
{
"stream_config" : "/path/to/streams_config.json",
"pattern_rows": 9,
"pattern_columns": 6,
"frame" : 0,
"streams": {
"camera1_rgb": null,
"camera2_thermal":
{
"threshold" : 155,
"closing" : 4,
"invert" : true
},
"camera3_nir":
{
"threshold" : 105,
"closing" : 2,
"invert" : false
}
},
"duplicate_calibration": {
"camera3_nir": "camera3_depth"
}
}
#!/usr/bin/env python
#
# Copyright (c) 2021 Idiap Research Institute, https://www.idiap.ch/
#
import argparse
import numpy as np
import copy
import json
import cv2
from .calibration import detect_chessboard_corners, preprocess_image
from bob.io.stream import Stream, StreamFile
from bob.io.image import bob_to_opencvbgr
pixels_xy = []
#def mlb_event(event, x, y, flags, param):
def point_picking_event_cb(event, x, y, flags, params):
"""
This function used as a callback when a mouse left button event is activated
by clicking on the opencv image. It saves the pixel selected in a global
list 'pixels_xy'.
"""
global pixels_xy
if event == cv2.EVENT_LBUTTONDOWN:
pixels_xy.append([x,y])
print("Point selected for marker {}: {}".format(len(pixels_xy), pixels_xy))
def warp_calibrate(h5_file, config, verbosity):
"""
This function load the streams saved in datasets of an hdf5 file, with images
of the chessboard targets. It select the frame defined, convert from bob to
grayscale <H,W>, process if necessary and try to detect a target. The four side
corners are saved as markers to be later used for linear warp calibration in
bob.ip.stereo.
If the target is not deteted, an interactive figure open, where the user
should click on the 4 corners (markers), in this order:
1. down-left
2. up-left
3. down-right
4. up-right
Parameters
----------
h5_file : :py:class:`str`
Hdf5 file path with captured image of the target in datasets.
config : :py:class:`dict`
A configuration to read the streams in the hdf5 file and run the target
detection.
Returns
-------
:py:class:`dict`
A dictionnary with detected markers for each streams.
"""
rows = config["pattern_rows"] - 1
columns = config["pattern_columns"] - 1
pattern_size = (rows, columns)
frame = config["frame"]
with StreamFile(h5_file, config["stream_config"]) as f:
calibration = {}
for stream_name, params in config["streams"].items():
# Load Stream, select frame, convert from bob to opencv grayscale
# and squeeze to 1
stream = Stream(stream_name, f)
image = stream.normalize()[frame]
if image.shape[0] == 3:
image = cv2.cvtColor(bob_to_opencvbgr(image), cv2.COLOR_BGR2GRAY)
image = image.squeeze()
if params is not None:
if params["invert"]:
image = np.ones_like(image)*255 - image
prep_image = preprocess_image(image, params["closing"], params["threshold"])
else:
prep_image = None
ret, ptrn_pts = detect_chessboard_corners(image, prep_image, pattern_size, verbosity)
if ret:
print("Stream : {} : Detected chessboard corners.".format(stream.name))
markers = np.stack([ptrn_pts[0][0],
ptrn_pts[rows-1][0],
ptrn_pts[rows*(columns-1)][0],
ptrn_pts[rows*columns -1][0]])
else:
print( "Stream : {} : Failed to detect chessboard corners.".format(stream.name))
window_name = "image_{}".format(stream.name)
cv2.imshow(window_name, image)
global pixels_xy
pixels_xy = []
cv2.setMouseCallback(window_name, point_picking_event_cb)
while True:
key = cv2.waitKey(1)
if len(pixels_xy) == 4:
print( "Stream : {} : Finished to collect manually\
the corners.".format(stream.name))
cv2.destroyAllWindows()
markers = np.stack(copy.copy(pixels_xy))
break
if key == 27 or cv2.getWindowProperty(window_name, cv2.WND_PROP_VISIBLE) < 1:
print( "Stream : {} : Failed to collect manually\
the corners.".format(stream.name))
cv2.destroyAllWindows()
markers = None
break
if markers is not None:
# As markers are ndarray, tolist() is used to serialized the values
calibration[stream.name] = {"markers": markers.tolist()}
return calibration
def fulfill_calibration(calibration, reference):
"""
This function fulfill the calibration keys to be compatible with
bob.ip.stereo standard calibration saved in JSON.
Parameters
----------
calibration : :py:class:`dict`
A dictionary with streams/warp-markers as keys/values.
reference : :py:class:`str`
reference stream.
Returns
-------
:py:class:`dict`
The fulfilled calibration.
"""
# TODO: To not duplicate, verify if the standard keys defined somewhere else.
# Maybe it should be defined in bob.ip.stereo.Camera.load_camera_config()
standard_calibration_keys = ["camera_matrix",
"distortion_coefs",
"markers",
"relative_rotation",
"relative_translation"]
# load calibration config
fulfill_calibration = copy.copy(calibration)
for camera, camera_config in calibration.items():
for key in standard_calibration_keys:
if key not in camera_config.keys():
# Projection markers as the 4 external corners
fulfill_calibration[camera][key] = None
# TODO: bob.ip.stereo currently only accept "nir_left" as a reference.
# https://gitlab.idiap.ch/bob/bob.ip.stereo/-/blob/calibration/bob/ip/stereo/camera.py#L24
# While not corrected, the reference is written as "nir_left", but won't
# be used for warp alignement anyway. Once solve, the reference will be
# saved correctly.
fulfill_calibration[camera]["reference"] = "nir_left"
# if "reference" not in camera_config.keys():
# updated_config[camera]["reference"] = reference
return fulfill_calibration
def parse_arguments():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-c",
"--config",
type=str,
help="An absolute path to the JSON file containing the configuration \
to run the warp calibration."
)
parser.add_argument(
"-r",
"--reference",
type=str,
default=None,
help="Reference stream."
)
parser.add_argument("-i", "--h5-file", type=str, default=None, help="Input hdf5 file.")
parser.add_argument(
"-o", "--output-file",
type=str,
default=None,
help="Output calibration JSON file."
)
parser.add_argument(
"-v",
"--verbosity",
action="count",
default=0,
help="Output verbosity: -v output calibration result, -vv output the dataframe, \
-vvv plots the target detection.",
)
return parser.parse_args()
def main():
args = parse_arguments()
with open(args.config) as f:
config = json.load(f)
calibration = warp_calibrate(args.h5_file, config, args.verbosity)
# Duplicate warp calibration for specified pair of stream with the same sensor, often the case
# in cameras with a depth stream created from infrared (NIR) sensor(s).
if "duplicate_calibration" in config.keys():
for stream, duplicate_stream in config["duplicate_calibration"].items():
calibration[duplicate_stream] = calibration[stream]
calibration = fulfill_calibration(calibration, args.reference)
print(calibration)
if args.output_file is not None:
with open(args.output_file, "w") as f:
json.dump(calibration, f, indent=4, sort_keys=True)
print("{} written.".format(args.output_file))
if __name__ == "__main__":
main()
......@@ -18,6 +18,7 @@ Users Guide
:maxdepth: 2
api
warp_calibration
.. todolist::
......
.. _bob.ip.stereo.warp_calibration:
----------------
Warp Calibration
----------------
Without depth map / 3D pointcloud, the remapping of streams on a reference is not useable. To align images in different streams from bounding boxes defined in the reference stream, a linear transform (warp) can be used.
To create such transform, it is necessary to collect four fixed points static in all streams. First, capture an hdf5 file with a chessboard target kept static during the whole capture, in front of the camears and at a defined distance.
This linear calibration is dependent of the distance camera-target, thus the result of an alignement will depend of this distance.
Configuration
-------------
Here is an example of JSON configuration to create for a warp calibration.
.. code-block:: json
{
"stream_config" : "/path/to/streams_config.json",
"pattern_rows": 10,
"pattern_columns": 7,
"frame" : 0,
"streams": {
"camera1_rgb": null,
"camera2_thermal":
{
"threshold" : 155,
"closing" : 4,
"invert" : true
},
"camera3_nir":
{
"threshold" : 105,
"closing" : 2,
"invert" : false
}
},
"duplicate_calibration": {
"camera3_nir": "camera3_depth"
}
}
| - ``stream_config`` : Path to the JSON data config.
| - ``pattern_rows/columns`` : Number of rows/columns in the chessboard.
| - ``frame`` : Frame number to select in the dataset.
| - ``streams`` : Streams to calibrate. It must correspond to the datasets in the hdf5 file.
| - ``threshold`` : Pixel value (0-255) used to binary threshold grayscale image.
| - ``closing`` : Closing size in pixel.
| - ``invert`` : Invert the pixel intensity (used for thermal images, when black square render white pixel under halogen illumination).
| - ``duplicate_calibration`` : To copy the calibration of one streams to another. Used by example for depth stream that is aligned with a NIR streams.
calibration
-----------
.. code-block:: sh
$ warp_calibration.py -i /path/to/capture.h5 -c warp_config.json -vvv -o /path/to/calibration.json
If the chessboard pattern is not found, the image is displayed. The user must click on the internal 4 side-corners of the chessboard in this order:
1. down-left (in blue)
2. up-left (still in blue)
3. down-right (in yellow)
4. up-right (in yellow)
.. image:: img/warp_calibration_corners.png
:width: 25 %
:align: center
If the user does not want to output markers, the window can be closed, the stream will not appear in the JSON output.
......@@ -129,6 +129,7 @@ setup(
'console_scripts': [
'stereo_matcher.py = bob.ip.stereo.stereo_matcher:main',
'calibration.py = bob.ip.stereo.calibration:main',
'warp_calibration.py = bob.ip.stereo.warp_calibration:main',
],
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment