一、简介
二、数据处理
三、PointNet++(SSG)网络搭建
四、训练、测试
在上一节点云处理:基于Paddle2.0实现PointNet++对点云进行分类处理①中,我们实现了PointNet++中比较重要的几个基础部分的搭建,包括Sampling层,Grouping层等的搭建,那么这一节我们将一起来实现PointNet++(SSG)整体网络搭建,并进行训练和测试。
!unzip data/data70460/shapenet_part_seg_hdf5_data.zip
!mv hdf5_data dataset
import os
import numpy as np
import random
import h5py
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from util.model import PointNetSetAbstraction #引入模块
train_list = ['ply_data_train0.h5', 'ply_data_train1.h5', 'ply_data_train2.h5', 'ply_data_train3.h5', 'ply_data_train4.h5', 'ply_data_train5.h5']
test_list = ['ply_data_test0.h5', 'ply_data_test1.h5']
val_list = ['ply_data_val0.h5']
def pointDataLoader(mode='train'):
path = './dataset/'
BATCHSIZE = 64
MAX_POINT = 1024
datas = []
labels = []
if mode == 'train':
for file_list in train_list:
f = h5py.File(os.path.join(path, file_list), 'r') #读取h5文件
#for key in f.keys():
#print("aaaaaaaaaaaaaaaaaaaaaaaa")
#print(f[key], key, f[key].name)
#break
datas.extend(f['data'][:, :MAX_POINT, :]) #(2048,1024,3)的列表
labels.extend(f['label']) #(2048,1)的列表
f.close()
elif mode == 'test':
for file_list in test_list:
f = h5py.File(os.path.join(path, file_list), 'r')
datas.extend(f['data'][:, :MAX_POINT, :])
labels.extend(f['label'])
f.close()
else:
for file_list in val_list:
f = h5py.File(os.path.join(path, file_list), 'r')
datas.extend(f['data'][:, :MAX_POINT, :])
labels.extend(f['label'])
f.close()
datas = np.array(datas)
labels = np.array(labels)
print('==========load over==========')
index_list = list(range(len(datas)))
def pointDataGenerator():
if mode == 'train':
random.shuffle(index_list)
datas_list = []
labels_list = []
for i in index_list:
datas_list.append(datas[i].T.astype('float32'))
labels_list.append(labels[i].astype('int64'))
if len(datas_list) == BATCHSIZE: #64个样本为一批
yield np.array(datas_list), np.array(labels_list)#(64,3,1024)(64,1)
datas_list = []
labels_list = []
if len(datas_list) > 0:
yield np.array(datas_list), np.array(labels_list)
return pointDataGenerator
补充注释:
1.
for key in f.keys():
print("aaaaaaaaaaaaaaaaaaaaaaaa")
print(f[key], key, f[key].name)
aaaaaaaaaaaaaaaaaaaaaaaadata /data label /label pid /pid
2.np.array(datas).shape np.array(labels).shape
训练时:
(2048, 1024, 3) (2048, 1) (4096, 1024, 3) (4096, 1) (6144, 1024, 3) (6144, 1) (8192, 1024, 3) (8192, 1) (10240, 1024, 3) (10240, 1) (12137, 1024, 3) (12137, 1)
测试时:
预测时:
(1870, 1024, 3) (1870, 1)
PointNet++(SSG)网络搭建过程:
1、首先将多个SetAbstraction层堆叠起来对点云进行特征提取,SetAbstraction层作用可以类比于图像处理中的卷积层,能够提取点云中局部的全局特征。(Figure1中的Part1)
2、然后最终的SetAbstraction层调用sample_and_group_all函数,作用可以类比于图像处理中的全局池化,将点云中的特征聚拢。(Figure1中的Part2)
3、最后经过一个mlp搭建分类最后一部分网络。(Figure1中的Part3)
class PointNet2(nn.Layer):
def __init__(self, num_class=16, normal_channel=False):#是否使用其他特征信息
super(PointNet2, self).__init__()
in_channel = 6 if normal_channel else 3
self.normal_channel = normal_channel
self.sa1 = PointNetSetAbstraction(npoint=512, radius=0.2, nsample=32, in_channel=in_channel, mlp=[64, 64, 128], group_all=False)
#Conv (3,64,1) (64,64,1) (64,128,1)
self.sa2 = PointNetSetAbstraction(npoint=128, radius=0.4, nsample=64, in_channel=128 + 3, mlp=[128, 128, 256], group_all=False)
#Conv (128,128,1) (128,128,1) (128,256,1)
self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=256 + 3, mlp=[256, 512, 1024], group_all=True)
#Conv (256,256,1) (256,512,1) (512,1024,1)
self.fc1 = nn.Linear(1024, 512)
self.bn1 = nn.BatchNorm1D(512)
self.drop1 = nn.Dropout(0.4)
self.fc2 = nn.Linear(512, 256)
self.bn2 = nn.BatchNorm1D(256)
self.drop2 = nn.Dropout(0.4)
self.fc3 = nn.Linear(256, num_class)
def forward(self, xyz): #(64,3,1024)
B, _, _ = xyz.shape #B=64
if self.normal_channel:
norm = xyz[:, 3:, :]
xyz = xyz[:, :3, :]
else:
norm = None
l1_xyz, l1_points = self.sa1(xyz, norm) #(64,3,512) (64,128,512)
l2_xyz, l2_points = self.sa2(l1_xyz, l1_points) #(64,3,128) (64,256,128)
l3_xyz, l3_points = self.sa3(l2_xyz, l2_points) #(64,3,1) (64,1024,1)
x = l3_points.reshape([B, 1024]) #(64,1024)
x = self.drop1(F.relu(self.bn1(self.fc1(x))))#(64,512)
x = self.drop2(F.relu(self.bn2(self.fc2(x))))#(64,256)
x = self.fc3(x) #(64,16)
x = F.log_softmax(x, -1)#(64,16)
return x
if __name__ == '__main__':
PointNet2 = PointNet2()
paddle.summary(PointNet2, (64, 3, 1024))
def train():
train_loader = pointDataLoader(mode='train')
val_loader = pointDataLoader(mode='val')
model = PointNet2()
model.train()
optim = paddle.optimizer.Adam(parameters=model.parameters(), weight_decay=0.001)
epoch_num = 2
for epoch in range(epoch_num):
# train
print("===================================train===========================================")
for batch_id, data in enumerate(train_loader()):
inputs = paddle.to_tensor(data[0]) #(64,3,1024)
labels = paddle.to_tensor(data[1])#(64,1)
predicts = model(inputs)#(64,16)
loss = F.nll_loss(predicts, labels)
acc = paddle.metric.accuracy(predicts, labels)
if batch_id % 10 == 0:
print("train: epoch: {}, batch_id: {}, loss is: {}, accuracy is: {}".format(epoch, batch_id, loss.numpy(), acc.numpy()))
loss.backward()
optim.step()
optim.clear_grad()
if epoch % 2 == 0:
paddle.save(model.state_dict(), './model/PointNet2.pdparams')
paddle.save(optim.state_dict(), './model/PointNet2.pdopt')
# validation
print("===================================val===========================================")
model.eval()
accuracies = []
losses = []
for batch_id, data in enumerate(val_loader()):
inputs = paddle.to_tensor(data[0])
labels = paddle.to_tensor(data[1])
predicts = model(inputs)
loss = F.nll_loss(predicts, labels)
acc = paddle.metric.accuracy(predicts, labels)
losses.append(loss.numpy())
accuracies.append(acc.numpy())
avg_acc, avg_loss = np.mean(accuracies), np.mean(losses)
print("validation: loss is: {}, accuracy is: {}".format(avg_loss, avg_acc))
model.train()
if __name__ == '__main__':
train()
PointNet++(MSG)网络是在PointNet++(SSG)网络基础上加入了多尺度特征提取策略,其中的MSG,即Multi-scale Grouping(多尺度)构建网络,是PointNet++网络基础版升级版。通过在同一PointNetSetAbstractionMsg层中,采取不同的尺度(Grouding中球形领域半径R的不同)对点云特征进行提取并concat起来,实现了多尺度特征提取,使得网络性能更好。
PointNet++(MSG)网络实现了在点云同一个FeatureMap中对多尺度的特征提取,采用不同尺度意味着不同大小的感受野,这个可以类比于GoogleNet的Inception结构。
PointNetSetAbstractionMsg层中多尺度特征提取依靠radius_list,radius_list输入的是一个list,里面对应的是球形领域中的不同半径,用于对不同的半径做Grouding,经过特征提取后,这样就能得到多尺度的特征,最终将不同半径下的点云特征(即不同尺度的特征)保存在new_points_list中,再最后将这些不同尺度的特征拼接到一起。
PointNetSetAbstractionMsg层
Input:
xyz: input points position data, [B, C, N]
points: input points data, [B, D, N]
Return:
new_xyz: sampled points position data, [B, C, S]
new_points_concat: sample points feature data, [B, D', S]
class PointNetSetAbstractionMsg(nn.Layer):
def __init__(self, npoint, radius_list, nsample_list, in_channel, mlp_list):
super(PointNetSetAbstractionMsg, self).__init__()
self.npoint = npoint
self.radius_list = radius_list
self.nsample_list = nsample_list
self.conv_blocks = []
self.bn_blocks = []
for i in range(len(mlp_list)):
convs = []
bns = []
last_channel = in_channel + 3
for out_channel in mlp_list[i]:
convs.append(nn.Conv2D(last_channel, out_channel, 1))
bns.append(nn.BatchNorm2D(out_channel))
last_channel = out_channel
self.conv_blocks.append(convs)
self.bn_blocks.append(bns)
def forward(self, xyz, points):
xyz = xyz.transpose([0, 2, 1]) #(B,N,3)
if points is not None:
points = points.transpose([0, 2, 1]) #(B,N,D)
B, N, C = xyz.shape
S = self.npoint
new_xyz = index_points(xyz, farthest_point_sample(xyz, S))#(B,S,3)
new_points_list = []
for i, radius in enumerate(self.radius_list):
K = self.nsample_list[i]
group_idx = query_ball_point(radius, K, xyz, new_xyz)
grouped_xyz = index_points(xyz, group_idx) #(64,S,K,C)
grouped_xyz -= new_xyz.reshape([B, S, 1, C])
if points is not None:
grouped_points = index_points(points, group_idx) #(64,S,K,D)
grouped_points = paddle.concat([grouped_points, grouped_xyz], axis=-1)
#(64,S,K,C+D)
else:
grouped_points = grouped_xyz#(64,S,K,C)
grouped_points = grouped_points.transpose([0, 3, 2, 1])
for j in range(len(self.conv_blocks[i])):
conv = self.conv_blocks[i][j]
bn = self.bn_blocks[i][j]
grouped_points = F.relu(bn(conv(grouped_points)))
new_points = paddle.max(grouped_points, 2) # [B, D', S]
new_points_list.append(new_points)
new_xyz = new_xyz.transpose([0, 2, 1])
new_points_concat = paddle.concat(new_points_list, axis=1)
return new_xyz, new_points_concat
class PointNet2(nn.Layer):
def __init__(self, num_class=16, normal_channel=False):
super(PointNet2, self).__init__()
in_channel = 3 if normal_channel else 0
self.normal_channel = normal_channel
self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], in_channel, [[32, 32, 64], [64, 64, 128], [64, 96, 128]])
#r=0.1 K=16 mlp=[32,32,64] -->(64,64,512)
#(64,3,1024) #r=0.2 K=32 mlp=[64,64,128] -->(64,128,512) (64,320,512)
#r=0.4 K=32 mlp=[64,96,128] -->(64,128,512)
#l1_xyz:(64,3,512) l1_points:(64,320,512)
self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320, [[64, 64, 128], [128, 128, 256], [128, 128, 256]])
#r=0.2 K=32 mlp=[64,64,128] -->(64,128,128)
#r=0.4 K=64 mlp=[128,128,256] -->(64,256,128) (64,640,128)
#r=0.8 K=128 mlp=[128,128,256] -->(64,256,128)
#l2_xyz:(64,3,128) l2_points:(64,640,128)
self.sa3 = PointNetSetAbstraction(None, None, None, 640 + 3, [256, 512, 1024], True)
#mlp=[256,512,1024] -->(64,1024,1)
#l3_xyz:(64,3,1) l3_points:(64,1024,1)
self.fc1 = nn.Linear(1024, 512)
self.bn1 = nn.BatchNorm1D(512)
self.drop1 = nn.Dropout(0.4)
self.fc2 = nn.Linear(512, 256)
self.bn2 = nn.BatchNorm1D(256)
self.drop2 = nn.Dropout(0.5)
self.fc3 = nn.Linear(256, num_class)
def forward(self, xyz): #(64,3,1024)
B, _, _ = xyz.shape
if self.normal_channel:#判断是否有其他特征信息
norm = xyz[:, 3:, :]
xyz = xyz[:, :3, :]
else:
norm = None
l1_xyz, l1_points = self.sa1(xyz, norm)#多尺度集合抽象层
l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)#多尺度集合抽象层
l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)#单尺度集合抽象层
x = l3_points.reshape([B, 1024]) #(64,1024)
x = self.drop1(F.relu(self.bn1(self.fc1(x)))) #(64,512)
x = self.drop2(F.relu(self.bn2(self.fc2(x)))) #(64,256)
x = self.fc3(x) #(64,16)
x = F.log_softmax(x, -1) #(64,16)
return x