使用opencv标定双目摄像头

在https://blog.csdn.net/weixin_39266208/article/details/120917840中，制作了一个9x12的棋盘图片，打印到A4纸上，每个格子长度22mm，用尺子测量，和预期一致，本文记录一下标定双目摄像头的过程。本文使用python，代码参考 OpenCV4快速入门 书中的c++代码。
OpenCV官方教程中也有对应的教程 https://docs.opencv.org/4.5.4/d6/d55/tutorial_table_of_content_calib3d.html

本文没有使用findChessboardCornersSB，而是使用findChessboardCorners+find4QuadCornerSubpix，因为使用的标定图片不匹配。

尝试阶段

打印棋盘中的角点

import cv2
cap = cv2.VideoCapture(2)
# 当前使用的摄像头 设置宽高，帧率会自动跟着变
cap.set(cv2.CAP_PROP_FRAME_WIDTH,2560)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 960)

fcc = cv2.VideoWriter_fourcc('M','J','P','G')
# 这里不设置默认会使用YUYV，速率会为5fps
cap.set(cv2.CAP_PROP_FOURCC, fcc)
count = 0
while cap.isOpened() and count < 60:
    ret, frame = cap.read()
    cv2.imshow('f',frame)
    k = cv2.waitKey(0)
    # ESC
    if k == 27:
        break
    # 空格
    if k == 32:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        #cv2.imwrite("calibration_{}.bmp".format(count), frame)
        count += 1
        l = frame[:,0:1280,:]
        lg = cv2.cvtColor(l, cv2.COLOR_BGR2GRAY)
        ret, corners = cv2.findChessboardCorners(lg, (8,11))
        ret, corners = cv2.find4QuadCornerSubpix(lg, corners, (7,7))
        cv2.drawChessboardCorners(l, (8,11), corners, True)
        cv2.imshow('l', l)
        cv2.waitKey()

cv2.destroyAllWindows()
cap.release()

正式阶段

第一次运行，抓取20张图片，以后为了方便，直接读取图片，第一次代码如下：

import cv2
cap = cv2.VideoCapture(2)
cap.set(cv2.CAP_PROP_FRAME_WIDTH,2560)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 960)

fcc = cv2.VideoWriter_fourcc('M','J','P','G')
# 这里不设置默认会使用YUYV，速率会为5fps
cap.set(cv2.CAP_PROP_FOURCC, fcc)
img_left = []
img_right = []


count = 0
while cap.isOpened():
    ret, frame = cap.read()
    cv2.imshow('f',frame)
    k = cv2.waitKey(1)
    # ESC
    if k == 27:
        break
    # SPACE
    # 确定标定图像完整再开始
    if k == 32:
        # 图像清晰再选中用于标定
        cv2.imshow('f',frame)
        if cv2.waitKey() == 32:
            # 保存用于以后使用
            cv2.imwrite("calibration_stereo/calibration_{}.bmp".format(count), frame)
            count += 1
            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            l = frame[:,0:1280]
            r = frame[:,1280:]
            img_left.append(l)
            img_right.append(r)

cv2.destroyAllWindows()
cap.release()

第二次运行获取图片的代码：

import cv2

img_left = []
img_right = []

cap = cv2.VideoCapture("calibration_stereo/calibration_%d.bmp", cv2.CAP_IMAGES)
while cap.isOpened():
    ret, frame = cap.read()
    if ret:
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        l = frame[:,0:1280]
        r = frame[:,1280:]
        img_left.append(l)
        img_right.append(r)
    else:
        break

cap.release()

完整代码

import cv2
img_left = []
img_right = []

cap = cv2.VideoCapture("文件的绝对路径/calibration_stereo/calibration_%d.bmp", cv2.CAP_IMAGES)
while cap.isOpened():
    ret, frame = cap.read()
    if ret:
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        l = frame[:,0:1280]
        r = frame[:,1280:]
        img_left.append(l)
        img_right.append(r)
    else:
        break

cap.release()

corners_left = []
corners_right = []

def get_corners(imgs, corners):
    for img in imgs:
        # 9x12棋盘有8x11个角点
        ret, c = cv2.findChessboardCorners(img, (8,11))
        assert(ret)
        ret, c = cv2.find4QuadCornerSubpix(img, c, (7,7))
        assert(ret)
        #cv2.drawChessboardCorners(img, (8,11), c, True)
        #print("debug conner:", c.shape, type(c), c.dtype)
        #cv2.imshow('t', img)
        #cv2.waitKey()
        #cv2.destroyWindow('t')
        corners.append(c)
 
print("获取角点", "left")
get_corners(img_left, corners_left)
print("获取角点", "right")
get_corners(img_right, corners_right)

import itertools
import numpy as np

points = np.zeros((8*11, 3), dtype=np.float32)

for y in range(11):
    for x in range(8):
        points[y*8+x][0] = 22. * x
        points[y*8+x][1] = 22. * y

object_points = list(itertools.repeat(points, len(img_left)))
cm_input = np.eye(3, dtype = np.float32)
print('calibration left')
imgsize = tuple(reversed(img_left[0].shape))
#ret = cv2.calibrateCameraROExtended(object_points, corners_left, imgsize, 7, cm_input, None, flags=cv2.CALIB_FIX_ASPECT_RATIO)

# 按照文档推荐，iFixedPoint使用topright，即7,如果使用0,则相当于calibrateCamera，[1, objectPoints[0].size()-2] 使用 object-releasing method，calibrateCamera速度很快，
# object-releasing method 速度比较慢，至于精度，我认为还是要通过测量已知坐标点，看谁误差小，否则不好说，
# 我使用object-releasing method得到的结果左右焦距摄像头在z方向的平移大约-7.68mm，快1cm了，
# 但实际的偏差根本没那么大，反而使用calibrateCamera算法，偏移1.45，表面看起来更真实， 另外
# 无意中发现（使用这个方法），如果图片数量比较少，误差会比较大，我使用3张图片和30张图片的结果
# 的畸变系数差的很多
# 带Extended和不带Extended的函数只是参数和返回值有微小差别，只有python版本有
ret = cv2.calibrateCameraROExtended(object_points, corners_left, imgsize, 7, cm_input, None)
#print(ret)
retval_l, cameraMatrix_l, distCoeffs_l, rvecs_l, tvecs_l, newObjPoints_l, stdDeviationsIntrinsics_l, stdDeviationsExtrinsics_l, stdDeviationsObjPoints_l, perViewErrors_l = ret
print('calibration right')
ret = cv2.calibrateCameraROExtended(object_points, corners_right, imgsize, 7, cm_input, None)
#print(ret)
retval_r, cameraMatrix_r, distCoeffs_r, rvecs_r, tvecs_r, newObjPoints_r, stdDeviationsIntrinsics_r, stdDeviationsExtrinsics_r, stdDeviationsObjPoints_r, perViewErrors_r = ret

# 按照文档推荐，先分别计算内参和畸变，然后使用CALIB_FIX_INTRINSIC，而不是在这一个函数中全计算出来
ret = cv2.stereoCalibrateExtended(object_points, corners_left,corners_right,cameraMatrix_l, distCoeffs_l, cameraMatrix_r, distCoeffs_r,imgsize, None, None, flags=cv2.CALIB_FIX_INTRINSIC)
#print(ret)
ret, _, _, _, _, R, T, E, F, perViewErrors =ret

print(cameraMatrix_l, "\n", distCoeffs_l,"\n", cameraMatrix_r, "\n",distCoeffs_r, "\n",R, T)



print("end")

输出如下：

获取角点 left
获取角点 right
calibration left
calibration right
[[853.41980694   0.         644.07745827]
 [  0.         853.61849384 509.05209825]
 [  0.           0.           1.        ]] 
 [[-0.06042329  0.2672132   0.00224203 -0.00108551 -0.29494271]] 
 [[830.29966566   0.         674.87185105]
 [  0.         828.23644547 477.0222712 ]
 [  0.           0.           1.        ]] 
 [[-0.03853824  0.13844433 -0.00205833  0.00337964 -0.12142663]] 
 [[ 9.99921954e-01 -3.89386209e-04 -1.24873492e-02]
 [ 6.42326374e-04  9.99794577e-01  2.02581043e-02]
 [ 1.24768958e-02 -2.02645442e-02  9.99716798e-01]] [[-120.91046143]
 [  -0.22335851]
 [  -7.67744893]]
end

验证结果

利用相机内参和双目左右摄像头之间的旋转平移矩阵对图像进行更正，使变换之后的图像的左右部分能够在一个平面上，并且按y坐标对齐，还是拿校准使用的图像来观察，看角点的y对齐了没有。

for i in range(len(img_left)):
    l = img_left[i]
    r = img_right[i]
    # 计算双目校正的矩阵
    R1, R2, P1, P2, Q, validPixROI1, validPixROI2 = cv2.stereoRectify(cameraMatrix_l, distCoeffs_l, cameraMatrix_r, distCoeffs_r, imgsize, R, T)
    # 计算校正后的映射关系
    maplx , maply = cv2.initUndistortRectifyMap(cameraMatrix_l, distCoeffs_l, R1, P1, imgsize, cv2.CV_16SC2)
    maprx , mapry = cv2.initUndistortRectifyMap(cameraMatrix_r, distCoeffs_r, R2, P2, imgsize, cv2.CV_16SC2)
    # 映射新图像
    lr = cv2.remap(l, maplx, maply, cv2.INTER_LINEAR)
    rr = cv2.remap(r, maprx, mapry, cv2.INTER_LINEAR)
    all = np.hstack((lr,rr))
    # all = np.hstack((l,r))
    # corners_left[i].shape: (88, 1, 2)
    # 变换之后和变换之前的角点坐标不一致，所以线不是正好经过角点，只是粗略估计，但偶尔能碰到离角点比较近的线，观察会比较明显
    cv2.line(all, (-1, int(corners_left[i][0][0][1])), (all.shape[1], int(corners_left[i][0][0][1])), (255), 1)

    # 可以看出左右图像y坐标对齐还是比较完美的，可以尝试着打印双目校正前的图片，很明显，左右y坐标是不对齐的
    cv2.imshow('a', all)
    c = cv2.waitKey()
    cv2.destroyAllWindows()
    if c == 27:
        break

校准前

校准后
可见，校准后y是对齐的很明显， 这样就可以做深度估计了。

我的测试结果表明，前面校准的参数中的7，改成0之后，换成其他图片测试，结果更准确，即对于我的情况object-releasing method 并没有比标准算法更好，具体的参见函数文档。