Open3d从RGBD生成点云、mesh

相机模型

$$x=PX\to x=K[R|t]X\to z_c\left[\begin{array}{c}u\\ v\\ 1\end{array}\right]=\left[\begin{array}{ccc}{k}_x& 0& u_0\\ 0& k_y& v_0\\ 0& 0& 1\end{array}\right][R|t]\left[\begin{array}{c}{x}_c\\ y_c\\ z_c\end{array}\right]$$
$x$表示图像点，$X$表示空间点。

$P$是正交矩阵，上三角矩阵$K$是相机内参数（相机硬件），矩阵内元素单位是像素（物理尺寸/单个CCD感光元宽度），$[u0,v0]$是光轴中心点图像坐标。

$[R|t]$是相机外参数，表示相机旋转和平移位姿。

python代码

import open3d as o3d
import cv2
import numpy as np

def RGBD2Point(depth_img, rgb_img):
    # 相机内参
    k_x = 5.775910000000000082e+02
    k_y = 5.787300000000000182e+02
    u_0 = 3.189049999999999727e+02
    v_0 = 2.426839999999999975e+02
    factor = 1
    K = [[k_x, 0, u_0], [0, k_y, v_0], [0, 0, 1]]

    # 逐点处理，此过程可以使用numpy优化
    m, n = depth_img.shape  # 480 640
    color_map, point_cloud = [], []
    for v in range(m):      # 行相当于y坐标
        for u in range(n):  # 列相当于x坐标
            if depth_img[v, u] == 0:
                continue
            rgb = rgb_img[v, u]
            rgb = [rgb[0], rgb[1], rgb[2]]
            rgb_info = np.array(rgb) / 255.0  # 颜色归一化到0-1之间
            rgb_info = rgb_info[::-1]  # cv2读取数据格式为BGR
            color_map.append(rgb_info)
            depth = depth_img[v, u]
            # 矩阵运算速度较慢
            # x_c, y_c, z_c = np.transpose(np.dot(np.linalg.inv(K), np.transpose(depth * np.array([u, v, 1]))))
            z_c = depth / factor
            x_c = (u - u_0) * z_c / k_x
            y_c = (v - v_0) * z_c / k_y
            point_cloud.append([x_c, y_c, z_c])
    point_cloud = np.array(point_cloud)
    color_map = np.array(color_map)
    return point_cloud, color_map  # shape都是(212342, 3) point为(x,y,z) color为(r,g,b)

s_depth = cv2.imread("data/depth.png", -1)  # 480 640
s_color = cv2.imread("data/rgb.jpg")     # (480, 640, 3)
points, color = RGBD2Point(s_depth, s_color)
pc = o3d.geometry.PointCloud()
pc.points = o3d.utility.Vector3dVector(points)
pc.colors = o3d.utility.Vector3dVector(color)
o3d.visualization.draw_geometries([pc])      # 可视化
o3d.io.write_point_cloud("data/pc.ply", pc)  # 保存文件

pcd.estimate_normals()
# estimate radius for rolling ball
distances = pcd.compute_nearest_neighbor_distance()
avg_dist = np.mean(distances)
radius = 1.5 * avg_dist
mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(pcd, o3d.utility.DoubleVector([radius, radius * 2]))
print(mesh.get_surface_area())   # 表面积：9609237.82040951
o3d.visualization.draw_geometries([mesh], window_name='Open3D downSample', width=800, height=600, left=50, top=50,
                                  point_show_normal=True, mesh_show_wireframe=True, mesh_show_back_face=True,)
# create the triangular mesh with the vertices and faces from open3d
tri_mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.triangles), vertex_normals=np.asarray(mesh.vertex_normals))
tri_mesh.show()

# a = [3, 5, 6]
# print(2 * a)  # [3, 5, 6, 3, 5, 6]
# print(2 * np.array(a))  # [6 10 12]

运行结果（左侧为生成的点云图，右图为原始RGB图）：

生成Mesh图失去了颜色信息，三角Mesh化结果不太行

注：本文图像数据来自论文DeepDeform