VoteNet源码解析
- 出发点：RPN(区域提议网络)系列用于点云检测的问题：RPN偏向于从目标中心预测，但是点云的目标中心不同于图像的是点云中心往往没有数据(零向量)，如果RPN基于中心预测，会产生零向量预测的问题
- 结构
a. Backbone：PointNet++作为点云特征提取器，由于其广泛应用于各类point-wise的特征抽取，由多个set-abstraction模块构成并带有upsampling模块，输出M个点，M中每个点云都是N中的子集，后面独立负责投票
b. proposal module：set abstraction layer with a post processing MLP2 to generate proposals after the max-pooling
c. Hough Voting Networks：xyz(3) + feature(256)
- 算法流程
a. 提取特征：借鉴PointNet++生成M个SeedPoint提取局部特征：从Nx3生成Mx(3+c)其中3为seed point的位置，C为点云的提取后的特征
b. 投票：每个seed point，经过MLP(fc, relu, batchnorm)生成vote，vote包含了每个seed的坐标偏移量和特征偏移量，以构成新的点云位置偏移和点云特征偏移，新的投票后生成的特征，在这里设置一个损失函数用于监督点云投票的位置偏移，这样设置损失函数的作用为-closer to each other than the seeds are（使同属于一个目标的点云相靠近），注意，这里的delta_x是有监督的利用GT的bbox来判断vote points是否属于某个目标，并作为中心
c. votes作为下一个检测模块的输入，首先使用FPS聚类方法得到K个聚类中心，然后以采样点固定半径为圆心组成group
d. 对每个group的向量都采用local normalized coordinate system（具体方法就是将所有的group_points减去中心坐标），将坐标归一化到中心点，这样能够提高点云的平移与旋转不变性，随后使用传统PointNet的处理方法对每个group并行地生成单一特征向量
e. 将生成的k个group单独生成regresstion和class的预测(最大池化点云群，后送入MLP)，生成object proposals：
Model
前向流程
class VoteNet(nn.Module):
r"""
A deep neural network for 3D object detection with end-to-end optimizable hough voting.
Parameters
----------
num_class: int
Number of semantics classes to predict over -- size of softmax classifier
num_heading_bin: int
num_size_cluster: int
input_feature_dim: (default: 0)
Input dim in the feature descriptor for each point. If the point cloud is Nx9, this
value should be 6 as in an Nx9 point cloud, 3 of the channels are xyz, and 6 are feature descriptors
num_proposal: int (default: 128)
Number of proposals/detections generated from the network. Each proposal is a 3D OBB with a semantic class.
vote_factor: (default: 1)
Number of votes generated from each seed point.
"""
def __init__(self, num_class, num_heading_bin, num_size_cluster, mean_size_arr,
input_feature_dim=0, num_proposal=128, vote_factor=1, sampling='vote_fps'):
super().__init__()
self.num_class = num_class
self.num_heading_bin = num_heading_bin
self.num_size_cluster = num_size_cluster
self.mean_size_arr = mean_size_arr
assert(mean_size_arr.shape[0] == self.num_size_cluster)
self.input_feature_dim = input_feature_dim
self.num_proposal = num_proposal
self.vote_factor = vote_factor
self.sampling=sampling
# Backbone point feature learning
self.backbone_net = Pointnet2Backbone(input_feature_dim=self.input_feature_dim)
# Hough voting, vote_factyor=1
self.vgen = VotingModule(self.vote_factor, 256)
# Vote aggregation and detection
self.pnet = ProposalModule(num_class, num_heading_bin, num_size_cluster,
mean_size_arr, num_proposal, sampling)
def forward(self, inputs):
end_points = {}
batch_size = inputs['point_clouds'].shape[0]
end_points = self.backbone_net(inputs['point_clouds'], end_points) # end_points:dict
# --------- HOUGH VOTING ---------
xyz = end_points['fp2_xyz']
features = end_points['fp2_features']
# seed
end_points['seed_inds'] = end_points['fp2_inds']
end_points['seed_xyz'] = xyz
end_points['seed_features'] = features
# votenet
xyz, features = self.vgen(xyz, features) # # [bs, vote, 3], [bs, 256, vote]， vote后的坐标和特征
# feature norm:按照1维度求二范数
features_norm = torch.norm(features, p=2, dim=1) # [bs, vote]，得到每个vote的二范数
features = features.div(features_norm.unsqueeze(1)) # [bs, 256, vote] / [bs, 1, vote]->[bs, 256, vote],div是除以每个vote的二范数，进行vote-wise的特征归一化
end_points['vote_xyz'] = xyz
end_points['vote_features'] = features
end_points = self.pnet(xyz, features, end_points)
return end_points
特征提取：backbone(PointNet++)
class Pointnet2Backbone(nn.Module):
# __init__部分省略
def forward(self, pointcloud: torch.cuda.FloatTensor, end_points=None):
if not end_points: end_points = {}
batch_size = pointcloud.shape[0]
xyz, features = self._break_up_pc(pointcloud)
# --------- 4 SET ABSTRACTION LAYERS ---------
# 分别是:采样后的位置，采样后的特征，每个采样点对应的周围相关点
xyz, features, fps_inds = self.sa1(xyz, features) # [bs, npoint, 3], (B, mlp[-1], npoint), [bs, npoint]，剩余的SA和FP相同思路
end_points['sa1_inds'] = fps_inds
end_points['sa1_xyz'] = xyz
end_points['sa1_features'] = features
xyz, features, fps_inds = self.sa2(xyz, features) # this fps_inds is just 0,1,...,1023
end_points['sa2_inds'] = fps_inds
end_points['sa2_xyz'] = xyz
end_points['sa2_features'] = features
xyz, features, fps_inds = self.sa3(xyz, features) # this fps_inds is just 0,1,...,511
end_points['sa3_xyz'] = xyz
end_points['sa3_features'] = features
xyz, features, fps_inds = self.sa4(xyz, features) # this fps_inds is just 0,1,...,255
end_points['sa4_xyz'] = xyz
end_points['sa4_features'] = features
# --------- 2 FEATURE UPSAMPLING LAYERS --------
features = self.fp1(end_points['sa3_xyz'], end_points['sa4_xyz'], end_points['sa3_features'], end_points['sa4_features'])
features = self.fp2(end_points['sa2_xyz'], end_points['sa3_xyz'], end_points['sa2_features'], features)
end_points['fp2_features'] = features
end_points['fp2_xyz'] = end_points['sa2_xyz']
num_seed = end_points['fp2_xyz'].shape[1]
end_points['fp2_inds'] = end_points['sa1_inds'][:,0:num_seed] # indices among the entire input point clouds
return end_points
投票：VotingModule
class VotingModule(nn.Module):
def __init__(self, vote_factor, seed_feature_dim):
""" Votes generation from seed point features.
Args:
vote_facotr: int
number of votes generated from each seed point
seed_feature_dim: int
number of channels of seed point features
vote_feature_dim: int
number of channels of vote features
"""
super().__init__()
self.vote_factor = vote_factor
self.in_dim = seed_feature_dim
self.out_dim = self.in_dim # due to residual feature, in_dim has to be == out_dim
self.conv1 = torch.nn.Conv1d(self.in_dim, self.in_dim, 1)
self.conv2 = torch.nn.Conv1d(self.in_dim, self.in_dim, 1)
self.conv3 = torch.nn.Conv1d(self.in_dim, (3+self.out_dim) * self.vote_factor, 1)
self.bn1 = torch.nn.BatchNorm1d(self.in_dim)
self.bn2 = torch.nn.BatchNorm1d(self.in_dim)
def forward(self, seed_xyz, seed_features):
""" Forward pass.
Arguments:
seed_xyz: (batch_size, num_seed, 3) Pytorch tensor
seed_features: (batch_size, feature_dim, num_seed) Pytorch tensor
Returns:
vote_xyz: (batch_size, num_seed*vote_factor, 3)
vote_features: (batch_size, vote_feature_dim, num_seed*vote_factor)
"""
# seeds 来自fp2
batch_size = seed_xyz.shape[0]
num_seed = seed_xyz.shape[1]
# vote_factor: vote的数量
num_vote = num_seed*self.vote_factor
net = F.relu(self.bn1(self.conv1(seed_features)))
net = F.relu(self.bn2(self.conv2(net)))
net = self.conv3(net) # (batch_size, (3+out_dim)*vote_factor, num_seed)
# [bs, seed, 1, 3+256=259]
net = net.transpose(2,1).view(batch_size, num_seed, self.vote_factor, 3+self.out_dim)
# 计算位置offset相加
offset = net[:,:,:,0:3]
vote_xyz = seed_xyz.unsqueeze(2) + offset
vote_xyz = vote_xyz.contiguous().view(batch_size, num_vote, 3) # [bs, vote, 3]
# 计算特征的offset并相加
residual_features = net[:,:,:,3:] # (batch_size, num_seed, vote_factor, out_dim)
vote_features = seed_features.transpose(2,1).unsqueeze(2) + residual_features
vote_features = vote_features.contiguous().view(batch_size, num_vote, self.out_dim)
vote_features = vote_features.transpose(2,1).contiguous() # [bs, 256, vote]
return vote_xyz, vote_features
Proposal阶段：
# votenet的第二阶段：object proposal and classification
class ProposalModule(nn.Module):
def __init__(self, num_class, num_heading_bin, num_size_cluster, mean_size_arr, num_proposal, sampling, seed_feat_dim=256):
super().__init__()
self.num_class = num_class
self.num_heading_bin = num_heading_bin
self.num_size_cluster = num_size_cluster
self.mean_size_arr = mean_size_arr
self.num_proposal = num_proposal
self.sampling = sampling
self.seed_feat_dim = seed_feat_dim
# Vote clustering
self.vote_aggregation = PointnetSAModuleVotes(
npoint=self.num_proposal, # 128
radius=0.3,
nsample=16,
mlp=[self.seed_feat_dim, 128, 128, 128],
use_xyz=True,
normalize_xyz=True
)
# Object proposal/detection
# Objectness scores (2), center residual (3),
# heading class+residual (num_heading_bin*2), size class+residual(num_size_cluster*4)
self.conv1 = torch.nn.Conv1d(128,128,1)
self.conv2 = torch.nn.Conv1d(128,128,1)
# out_dim=2+3+12*2+10*4+10：分别是中心度、目标中心、偏航角bin与其residual，尺寸大小及其residual，种类
self.conv3 = torch.nn.Conv1d(128,2+3+num_heading_bin*2+num_size_cluster*4+self.num_class,1)
self.bn1 = torch.nn.BatchNorm1d(128)
self.bn2 = torch.nn.BatchNorm1d(128)
def forward(self, xyz, features, end_points):
"""
Args:
xyz: (B,K,3)
features: (B,C,K)
Returns:
scores: (B,num_proposal,2+3+NH*2+NS*4)
"""
# 默认是vote_fps
if self.sampling == 'vote_fps':
# Farthest point sampling (FPS) on votes
xyz, features, fps_inds = self.vote_aggregation(xyz, features)
sample_inds = fps_inds
elif self.sampling == 'seed_fps':
# FPS on seed and choose the votes corresponding to the seeds
# This gets us a slightly better coverage of *object* votes than vote_fps (which tends to get more cluster votes)
sample_inds = pointnet2_utils.furthest_point_sample(end_points['seed_xyz'], self.num_proposal)
xyz, features, _ = self.vote_aggregation(xyz, features, sample_inds)
elif self.sampling == 'random':
# Random sampling from the votes
num_seed = end_points['seed_xyz'].shape[1]
batch_size = end_points['seed_xyz'].shape[0]
sample_inds = torch.randint(0, num_seed, (batch_size, self.num_proposal), dtype=torch.int).cuda()
xyz, features, _ = self.vote_aggregation(xyz, features, sample_inds)
else:
log_string('Unknown sampling strategy: %s. Exiting!'%(self.sampling))
exit()
# 这是聚类后的proposal及其索引
end_points['aggregated_vote_xyz'] = xyz # (batch_size, num_proposal, 3)
end_points['aggregated_vote_inds'] = sample_inds # (batch_size, num_proposal,) # should be 0,1,2,...,num_proposal
# --------- PROPOSAL GENERATION ---------
net = F.relu(self.bn1(self.conv1(features)))
net = F.relu(self.bn2(self.conv2(net)))
net = self.conv3(net) # (batch_size, 2+3+num_heading_bin*2+num_size_cluster*4, num_proposal)
end_points = decode_scores(net, end_points, self.num_class, self.num_heading_bin, self.num_size_cluster, self.mean_size_arr)
return end_points
decode_scores:生成最终结果
votenet采用的是：bin-based-detection这类方法，具体方法可以参考PointRCNN，将回归问题分解为分类+回归问题，相对直接回归提高了准确率
# 生成最终预测结果
def decode_scores(net, end_points, num_class, num_heading_bin, num_size_cluster, mean_size_arr):
# net:(batch_size, 2+3+num_heading_bin*2+num_size_cluster*4, num_proposal)
net_transposed = net.transpose(2,1) # (batch_size, 1024, ..)
batch_size = net_transposed.shape[0]
num_proposal = net_transposed.shape[1]
# 置信度
objectness_scores = net_transposed[:,:,0:2] # pos & neg
end_points['objectness_scores'] = objectness_scores
# proposal的原始中心位置(base xyz)和通过proposal feature经过mlp预测的offset
base_xyz = end_points['aggregated_vote_xyz'] # (batch_size, num_proposal, 3)
center = base_xyz + net_transposed[:,:,2:5] # (batch_size, num_proposal, 3)
end_points['center'] = center
# yaw值计算
heading_scores = net_transposed[:,:,5:5+num_heading_bin] # head_bin=12,代表12个不同的方向
heading_residuals_normalized = net_transposed[:,:,5+num_heading_bin:5+num_heading_bin*2]
end_points['heading_scores'] = heading_scores # Bxnum_proposalxnum_heading_bin
end_points['heading_residuals_normalized'] = heading_residuals_normalized # Bxnum_proposalxnum_heading_bin (should be -1 to 1)
end_points['heading_residuals'] = heading_residuals_normalized * (np.pi/num_heading_bin) # Bxnum_proposalxnum_heading_bin
# size计算
size_scores = net_transposed[:,:,5+num_heading_bin*2:5+num_heading_bin*2+num_size_cluster]
size_residuals_normalized = net_transposed[:,:,5+num_heading_bin*2+num_size_cluster:5+num_heading_bin*2+num_size_cluster*4].view([batch_size, num_proposal, num_size_cluster, 3]) # Bxnum_proposalxnum_size_clusterx3
end_points['size_scores'] = size_scores
end_points['size_residuals_normalized'] = size_residuals_normalized
end_points['size_residuals'] = size_residuals_normalized * torch.from_numpy(mean_size_arr.astype(np.float32)).cuda().unsqueeze(0).unsqueeze(0)
# point-wise的类别计算
sem_cls_scores = net_transposed[:,:,5+num_heading_bin*2+num_size_cluster*4:] # Bxnum_proposalx10
end_points['sem_cls_scores'] = sem_cls_scores
return end_points
Loss计算
def get_loss(end_points, config):
""" Loss functions
Args:
end_points: dict
{
seed_xyz, seed_inds, vote_xyz,
center,
heading_scores, heading_residuals_normalized,
size_scores, size_residuals_normalized,
sem_cls_scores, #seed_logits,#
center_label,
heading_class_label, heading_residual_label,
size_class_label, size_residual_label,
sem_cls_label,
box_label_mask,
vote_label, vote_label_mask
}
config: dataset config instance
Returns:
loss: pytorch scalar tensor
end_points: dict
"""
# Vote loss
vote_loss = compute_vote_loss(end_points)
end_points['vote_loss'] = vote_loss
# Obj loss
objectness_loss, objectness_label, objectness_mask, object_assignment = \
compute_objectness_loss(end_points)
end_points['objectness_loss'] = objectness_loss
end_points['objectness_label'] = objectness_label
end_points['objectness_mask'] = objectness_mask
end_points['object_assignment'] = object_assignment
total_num_proposal = objectness_label.shape[0]*objectness_label.shape[1]
end_points['pos_ratio'] = \
torch.sum(objectness_label.float().cuda())/float(total_num_proposal)
end_points['neg_ratio'] = \
torch.sum(objectness_mask.float())/float(total_num_proposal) - end_points['pos_ratio']
# Box loss and sem cls loss
center_loss, heading_cls_loss, heading_reg_loss, size_cls_loss, size_reg_loss, sem_cls_loss = \
compute_box_and_sem_cls_loss(end_points, config)
end_points['center_loss'] = center_loss
end_points['heading_cls_loss'] = heading_cls_loss
end_points['heading_reg_loss'] = heading_reg_loss
end_points['size_cls_loss'] = size_cls_loss
end_points['size_reg_loss'] = size_reg_loss
end_points['sem_cls_loss'] = sem_cls_loss
box_loss = center_loss + 0.1*heading_cls_loss + heading_reg_loss + 0.1*size_cls_loss + size_reg_loss
end_points['box_loss'] = box_loss
# Final loss function
loss = vote_loss + 0.5*objectness_loss + box_loss + 0.1*sem_cls_loss
loss *= 10
end_points['loss'] = loss
# --------------------------------------------
# Some other statistics
obj_pred_val = torch.argmax(end_points['objectness_scores'], 2) # B,K
obj_acc = torch.sum((obj_pred_val==objectness_label.long()).float()*objectness_mask)/(torch.sum(objectness_mask)+1e-6)
end_points['obj_acc'] = obj_acc
return loss, end_points