[calibration] Warp calibration script to save markers in config

- used for warp StreamFilter
- rst documentation to create a JSON config and run the script

bob/ip/stereo/config/warp_config.json

+{
+    "stream_config" : "/path/to/streams_config.json",
+    "pattern_rows": 9,
+    "pattern_columns": 6,
+    "frame" : 0,
+    "streams": {
+        "camera1_rgb": null,
+        "camera2_thermal":
+        {
+            "threshold" : 155,
+            "closing" : 4,
+            "invert" : true
+        },
+        "camera3_nir":
+        {
+            "threshold" : 105,
+            "closing" : 2,
+            "invert" : false
+        }
+    },
+    "duplicate_calibration": {
+        "camera3_nir": "camera3_depth"
+    }
+}

bob/ip/stereo/warp_calibration.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2021 Idiap Research Institute, https://www.idiap.ch/
+#
+
+import argparse
+import numpy as np
+import copy
+import json
+import cv2
+from .calibration import detect_chessboard_corners, preprocess_image
+from bob.io.stream import Stream, StreamFile
+from bob.io.image import bob_to_opencvbgr
+
+pixels_xy = []
+#def mlb_event(event, x, y, flags, param):
+def point_picking_event_cb(event, x, y, flags, params):
+    """
+    This function used as a callback when a mouse left button event is activated
+    by clicking on the opencv image. It saves the pixel selected in a global
+    list 'pixels_xy'.
+    """
+    global pixels_xy
+    if event == cv2.EVENT_LBUTTONDOWN:
+        pixels_xy.append([x,y])
+        print("Point selected for marker {}: {}".format(len(pixels_xy), pixels_xy))
+
+def warp_calibrate(h5_file, config, verbosity):
+    """
+    This function load the streams saved in datasets of an hdf5 file, with images
+    of the chessboard targets. It select the frame defined, convert from bob to
+    grayscale <H,W>, process if necessary and try to detect a target. The four side
+    corners are saved as markers to be later used for linear warp calibration in
+    bob.ip.stereo.
+    If the target is not deteted, an interactive figure open, where the user
+    should click on the 4 corners (markers), in this order:
+        1. down-left
+        2. up-left
+        3. down-right
+        4. up-right
+
+    Parameters
+    ----------
+    h5_file : :py:class:`str`
+        Hdf5 file path with captured image of the target in datasets.
+    config : :py:class:`dict`
+        A configuration to read the streams in the hdf5 file and run the target
+        detection.
+
+    Returns
+    -------
+    :py:class:`dict`
+        A dictionnary with detected markers for each streams.
+    """
+    rows = config["pattern_rows"] - 1
+    columns = config["pattern_columns"] - 1
+    pattern_size = (rows, columns)
+    frame = config["frame"]
+
+    with StreamFile(h5_file, config["stream_config"]) as f:
+        calibration = {}
+        for stream_name, params in config["streams"].items():
+            # Load Stream, select frame, convert from bob to opencv grayscale
+            # and squeeze to 1
+            stream = Stream(stream_name, f)
+            image = stream.normalize()[frame]
+            if image.shape[0] == 3:
+                image = cv2.cvtColor(bob_to_opencvbgr(image), cv2.COLOR_BGR2GRAY)
+            image = image.squeeze()
+
+            if params is not None:
+                if params["invert"]:
+                    image = np.ones_like(image)*255 - image
+                prep_image = preprocess_image(image, params["closing"], params["threshold"])
+            else:
+                prep_image = None
+            ret, ptrn_pts = detect_chessboard_corners(image, prep_image, pattern_size, verbosity)
+
+            if ret:
+                print("Stream : {} : Detected chessboard corners.".format(stream.name))
+                markers = np.stack([ptrn_pts[0][0],
+                                    ptrn_pts[rows-1][0],
+                                    ptrn_pts[rows*(columns-1)][0],
+                                    ptrn_pts[rows*columns -1][0]])
+            else:
+                print( "Stream : {} : Failed to detect chessboard corners.".format(stream.name))
+                window_name = "image_{}".format(stream.name)
+                cv2.imshow(window_name, image)
+                global pixels_xy
+                pixels_xy = []
+                cv2.setMouseCallback(window_name, point_picking_event_cb)
+                while True:
+                    key = cv2.waitKey(1)
+                    if len(pixels_xy) == 4:
+                        print( "Stream : {} : Finished to collect manually\
+                                the corners.".format(stream.name))
+                        cv2.destroyAllWindows()
+                        markers = np.stack(copy.copy(pixels_xy))
+                        break
+                    if key == 27 or cv2.getWindowProperty(window_name, cv2.WND_PROP_VISIBLE) < 1:
+                        print( "Stream : {} : Failed to collect manually\
+                                the corners.".format(stream.name))
+                        cv2.destroyAllWindows()
+                        markers = None
+                        break
+
+            if markers is not None:
+                # As markers are ndarray, tolist() is used to serialized the values
+                calibration[stream.name] = {"markers": markers.tolist()}
+
+    return calibration
+
+def fulfill_calibration(calibration, reference):
+    """
+    This function fulfill the calibration keys to be compatible with
+    bob.ip.stereo standard calibration saved in JSON.
+
+    Parameters
+    ----------
+    calibration : :py:class:`dict`
+        A dictionary with streams/warp-markers as keys/values.
+    reference : :py:class:`str`
+        reference stream.
+
+    Returns
+    -------
+    :py:class:`dict`
+        The fulfilled calibration.
+    """
+    # TODO: To not duplicate, verify if the standard keys defined somewhere else.
+    # Maybe it should be defined in bob.ip.stereo.Camera.load_camera_config()
+    standard_calibration_keys = ["camera_matrix",
+                                 "distortion_coefs",
+                                 "markers",
+                                 "relative_rotation",
+                                 "relative_translation"]
+    # load calibration config
+    fulfill_calibration = copy.copy(calibration)
+    for camera, camera_config in calibration.items():
+        for key in standard_calibration_keys:
+            if key not in camera_config.keys():
+                # Projection markers as the 4 external corners
+                fulfill_calibration[camera][key] = None
+        # TODO: bob.ip.stereo currently only accept "nir_left" as a reference.
+        # https://gitlab.idiap.ch/bob/bob.ip.stereo/-/blob/calibration/bob/ip/stereo/camera.py#L24
+        # While not corrected, the reference is written as "nir_left", but won't
+        # be used for warp alignement anyway. Once solve, the reference will be
+        # saved correctly.
+        fulfill_calibration[camera]["reference"] = "nir_left"
+        # if "reference" not in camera_config.keys():
+        #     updated_config[camera]["reference"] = reference
+    return fulfill_calibration
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        help="An absolute path to the JSON file containing the configuration \
+              to run the warp calibration."
+    )
+    parser.add_argument(
+        "-r",
+        "--reference",
+        type=str,
+        default=None,
+        help="Reference stream."
+    )
+    parser.add_argument("-i", "--h5-file", type=str, default=None, help="Input hdf5 file.")
+    parser.add_argument(
+        "-o", "--output-file",
+        type=str,
+        default=None,
+        help="Output calibration JSON file."
+    )
+    parser.add_argument(
+        "-v",
+        "--verbosity",
+        action="count",
+        default=0,
+        help="Output verbosity: -v output calibration result, -vv output the dataframe, \
+                                -vvv plots the target detection.",
+    )
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    with open(args.config) as f:
+        config = json.load(f)
+
+    calibration = warp_calibrate(args.h5_file, config, args.verbosity)
+
+    # Duplicate warp calibration for specified pair of stream with the same sensor, often the case
+    # in cameras with a depth stream created from infrared (NIR) sensor(s).
+    if "duplicate_calibration" in config.keys():
+        for stream, duplicate_stream in config["duplicate_calibration"].items():
+            calibration[duplicate_stream] = calibration[stream]
+
+    calibration = fulfill_calibration(calibration, args.reference)
+
+    print(calibration)
+    if args.output_file is not None:
+        with open(args.output_file, "w") as f:
+            json.dump(calibration, f, indent=4, sort_keys=True)
+        print("{} written.".format(args.output_file))
+
+if __name__ == "__main__":
+    main()

doc/index.rst

@@ -18,6 +18,7 @@ Users Guide
    :maxdepth: 2
 
    api
+   warp_calibration
 
 .. todolist::
 

doc/warp_calibration.rst

+.. _bob.ip.stereo.warp_calibration:
+
+----------------
+Warp Calibration
+----------------
+
+Without depth map / 3D pointcloud, the remapping of streams on a reference is not useable. To align images in different streams from bounding boxes defined in the reference stream, a linear transform (warp) can be used.
+
+To create such transform, it is necessary to collect four fixed points static in all streams. First, capture an hdf5 file with a chessboard target kept static during the whole capture, in front of the camears and at a defined distance.
+
+This linear calibration is dependent of the distance camera-target, thus the result of an alignement will depend of this distance.
+
+Configuration
+-------------
+
+Here is an example of JSON configuration to create for a warp calibration.
+
+.. code-block:: json
+
+    {
+        "stream_config" : "/path/to/streams_config.json",
+        "pattern_rows": 10,
+        "pattern_columns": 7,
+        "frame" : 0,
+        "streams": {
+            "camera1_rgb": null,
+            "camera2_thermal":
+            {
+                "threshold" : 155,
+                "closing" : 4,
+                "invert" : true
+            },
+            "camera3_nir":
+            {
+                "threshold" : 105,
+                "closing" : 2,
+                "invert" : false
+            }
+        },
+        "duplicate_calibration": {
+            "camera3_nir": "camera3_depth"
+        }
+    }
+
+| - ``stream_config`` : Path to the JSON data config.
+| - ``pattern_rows/columns`` : Number of rows/columns in the chessboard.
+| - ``frame`` : Frame number to select in the dataset.
+| - ``streams`` : Streams to calibrate. It must correspond to the datasets in the hdf5 file.
+|   - ``threshold`` : Pixel value (0-255) used to binary threshold grayscale image.
+|   - ``closing`` : Closing size in pixel.
+|   - ``invert`` : Invert the pixel intensity (used for thermal images, when black square render white pixel under halogen illumination).
+| - ``duplicate_calibration`` : To copy the calibration of one streams to another. Used by example for depth stream that is aligned with a NIR streams.
+
+
+calibration
+-----------
+
+.. code-block:: sh
+
+    $ warp_calibration.py -i /path/to/capture.h5 -c warp_config.json -vvv -o /path/to/calibration.json
+
+If the chessboard pattern is not found, the image is displayed. The user must click on the internal 4 side-corners of the chessboard in this order:
+
+1. down-left (in blue)
+2. up-left (still in blue)
+3. down-right (in yellow)
+4. up-right (in yellow)
+
+.. image:: img/warp_calibration_corners.png
+   :width: 25 %
+   :align: center
+
+If the user does not want to output markers, the window can be closed, the stream will not appear in the JSON output.
+

setup.py

@@ -129,6 +129,7 @@ setup(
       'console_scripts': [
         'stereo_matcher.py = bob.ip.stereo.stereo_matcher:main',
         'calibration.py = bob.ip.stereo.calibration:main',
+        'warp_calibration.py = bob.ip.stereo.warp_calibration:main',
       ],
     },