light_openpose代码
关键点识别
推荐可以看一下这个视频
1.数据读取
读取coco关键点数据
coco.py
注意原本关键点是17个，在 datasets.transformations import ConvertKeypoints中重新生成了一个Neck，所以经过transformer过后的关键点是18个
# 关键点处理
def _convert(self, keypoints, w, h):
# Nose, Neck, R hand, L hand, R leg, L leg, Eyes, Ears
reorder_map = [1, 7, 9, 11, 6, 8, 10, 13, 15, 17, 12, 14, 16, 3, 2, 5, 4]
# print(f'len_reorder:{len(reorder_map)}')
# 重新排列关键点位置[ Nose, Neck, R hand, L hand, R leg, L leg, Eyes, Ears ]
converted_keypoints = list(keypoints[i - 1] for i in reorder_map)
# print(f'ck:{converted_keypoints}')
converted_keypoints.insert(1, [(keypoints[5][0] + keypoints[6][0]) / 2,
(keypoints[5][1] + keypoints[6][1]) / 2, 0]) # Add neck as a mean of shoulders
# 同时在1处插入neck作为shoulders
# print(f'con:{converted_keypoints}')
# 点1应和5,6密切相关，因为是他们的平均值
if keypoints[5][2] == 2 or keypoints[6][2] == 2:
converted_keypoints[1][2] = 2
elif keypoints[5][2] == 1 and keypoints[6][2] == 1:
converted_keypoints[1][2] = 1
if (converted_keypoints[1][0] < 0
or converted_keypoints[1][0] >= w
or converted_keypoints[1][1] < 0
or converted_keypoints[1][1] >= h):
converted_keypoints[1][2] = 2
return converted_keypoints
主要介绍2个函数
这个函数可以看这个链接heatmap
_add_gaussian
# 未标注或者关键点不可见
'''
添加高斯热图
以(x,y)为圆心向周围扩散
sigma 决定了高斯分布的拖尾长度
'''
def _add_gaussian(self, keypoint_map, x, y, stride, sigma):
n_sigma = 4
tl = [int(x - n_sigma * sigma), int(y - n_sigma * sigma)]
tl[0] = max(tl[0], 0)
tl[1] = max(tl[1], 0)
br = [int(x + n_sigma * sigma), int(y + n_sigma * sigma)]
map_h, map_w = keypoint_map.shape
br[0] = min(br[0], map_w * stride)
br[1] = min(br[1], map_h * stride)
shift = stride / 2 - 0.5
for map_y in range(tl[1] // stride, br[1] // stride):
for map_x in range(tl[0] // stride, br[0] // stride):
d2 = (map_x * stride + shift - x) * (map_x * stride + shift - x) + \
(map_y * stride + shift - y) * (map_y * stride + shift - y)
exponent = d2 / 2 / sigma / sigma
if exponent > 4.6052: # threshold, ln(100), ~0.01
continue
keypoint_map[map_y, map_x] += math.exp(-exponent)
if keypoint_map[map_y, map_x] > 1:
keypoint_map[map_y, map_x] = 1
第二个函数_set_paf
# 设置两个关节点之间的连接信息
def _set_paf(self, paf_map, x_a, y_a, x_b, y_b, stride, thickness):
x_a /= stride
y_a /= stride
x_b /= stride
y_b /= stride
# 得到两个点的向量
x_ba = x_b - x_a
y_ba = y_b - y_a
_, h_map, w_map = paf_map.shape
x_min = int(max(min(x_a, x_b) - thickness, 0))
x_max = int(min(max(x_a, x_b) + thickness, w_map))
y_min = int(max(min(y_a, y_b) - thickness, 0))
y_max = int(min(max(y_a, y_b) + thickness, h_map))
# 二者之间距离
norm_ba = (x_ba * x_ba + y_ba * y_ba) ** 0.5
if norm_ba < 1e-7: # Same points, no paf
return
# 单位向量
x_ba /= norm_ba
y_ba /= norm_ba
# 在图上显示与 limb 向量距离小于 thre 的点
# x-a ---> x-b
for y in range(y_min, y_max):
for x in range(x_min, x_max):
x_ca = x - x_a
y_ca = y - y_a
d = math.fabs(x_ca * y_ba - y_ca * x_ba)
if d <= thickness:
paf_map[0, y, x] = x_ba
paf_map[1, y, x] = y_ba
2.关键点识别后处理部分
在demo.py中
调用net得到网络生成结果
heatmaps是[128,136,19]的特征图
pafs是[]128,136,38]的特征图
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu)
# 记录总共检测到的关键点个数
total_keypoints_num = 0
# [x,y,conf,id]
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
# 提取第i个关键点的个数,把关键点的[x,y,conf,id]加入到all_keypoints_by_type
# 关键点个数使用total_keypoints_num记录
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
第一个重点函数extract_keypoints
def extract_keypoints(heatmap, all_keypoints, total_keypoint_num):
heatmap[heatmap < 0.1] = 0
# 添加边框
heatmap_with_borders = np.pad(heatmap, [(2, 2), (2, 2)], mode='constant')
# 取中心图，也就是去除周围一圈
heatmap_center = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 1:heatmap_with_borders.shape[1]-1]
# 取右边的图，左边一圈切掉
heatmap_left = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 2:heatmap_with_borders.shape[1]]
# 右边切掉
heatmap_right = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 0:heatmap_with_borders.shape[1]-2]
# 上边切掉
heatmap_up = heatmap_with_borders[2:heatmap_with_borders.shape[0], 1:heatmap_with_borders.shape[1]-1]
# 下边切掉
heatmap_down = heatmap_with_borders[0:heatmap_with_borders.shape[0]-2, 1:heatmap_with_borders.shape[1]-1]
# 将center图与其他几幅图做比较，如果center都大于left,right,up,down,说明这个点是峰值
heatmap_peaks = (heatmap_center > heatmap_left) &\
(heatmap_center > heatmap_right) &\
(heatmap_center > heatmap_up) &\
(heatmap_center > heatmap_down)
heatmap_peaks = heatmap_peaks[1:heatmap_center.shape[0]-1, 1:heatmap_center.shape[1]-1]
# 返回数组a中非零元素的索引值数组。
# 即返回可能是关键点的峰值最大的位置
keypoints = list(zip(np.nonzero(heatmap_peaks)[1], np.nonzero(heatmap_peaks)[0])) # (w, h)
# 排序
keypoints = sorted(keypoints, key=itemgetter(0))
suppressed = np.zeros(len(keypoints), np.uint8)
keypoints_with_score_and_id = []
keypoint_num = 0
for i in range(len(keypoints)):
# 判断第ige关键点是否是重复的
if suppressed[i]:
continue
for j in range(i+1, len(keypoints)):
# 如果两个点的位置接近，认为是同一个关键点
if math.sqrt((keypoints[i][0] - keypoints[j][0]) ** 2 +
(keypoints[i][1] - keypoints[j][1]) ** 2) < 6:
suppressed[j] = 1
keypoint_with_score_and_id = (keypoints[i][0], keypoints[i][1], heatmap[keypoints[i][1], keypoints[i][0]],
total_keypoint_num + keypoint_num)
keypoints_with_score_and_id.append(keypoint_with_score_and_id)
keypoint_num += 1
# 添加提取的关节点和置信度[x,y,conf,id]
all_keypoints.append(keypoints_with_score_and_id)
return keypoint_num
test.py来可视化center的部分
可以看出，最后得到一个峰值点，其他的都被抑制下去了
import numpy as np
import math
from matplotlib import pyplot as plt
# 产生高斯图
x, y = 10, 10
heatmap = np.zeros((20, 20))
xmin, xmax = x-4, x+4
ymin, ymax = y-4, y+4
sigma = 2
for map_y in range(ymin, ymax):
for map_x in range(xmin, xmax):
d2 = (map_x - x) * (map_x - x) + \
(map_y - y) * (map_y - y)
exponent = d2 / 2 / sigma / sigma
# if exponent > 4.6052: # threshold, ln(100), ~0.01
# continue
heatmap[map_y, map_x] += math.exp(-exponent)
if heatmap[map_y, map_x] > 1:
heatmap[map_y, map_x] = 1
heatmap[heatmap < 0.1] = 0
heatmap_with_borders = np.pad(heatmap, [(2, 2), (2, 2)], mode='constant')
heatmap_center = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 1:heatmap_with_borders.shape[1]-1]
heatmap_left = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 2:heatmap_with_borders.shape[1]]
heatmap_right = heatmap_with_borders[1:heatmap_with_borders.shape[0]-1, 0:heatmap_with_borders.shape[1]-2]
heatmap_up = heatmap_with_borders[2:heatmap_with_borders.shape[0], 1:heatmap_with_borders.shape[1]-1]
heatmap_down = heatmap_with_borders[0:heatmap_with_borders.shape[0]-2, 1:heatmap_with_borders.shape[1]-1]
heatmap_peaks = (heatmap_center > heatmap_left) &\
(heatmap_center > heatmap_right) &\
(heatmap_center > heatmap_up) &\
(heatmap_center > heatmap_down)
print(heatmap_with_borders.shape,heatmap_center.shape,heatmap_left.shape,heatmap_right.shape,heatmap_down.shape)
heatmap_peaks = heatmap_peaks[1:heatmap_center.shape[0]-1, 1:heatmap_center.shape[1]-1]
print(heatmap_with_borders.shape)
plt.subplot(2,4,1)
plt.imshow(heatmap_with_borders)
plt.subplot(2,4,2)
plt.imshow(heatmap_center)
plt.subplot(2,4,3)
plt.imshow(heatmap_left)
plt.subplot(2,4,4)
plt.imshow(heatmap_right)
plt.subplot(2,4,5)
plt.imshow(heatmap_up)
plt.subplot(2,4,6)
plt.imshow(heatmap_down)
plt.subplot(2,4,7)
plt.imshow(heatmap_peaks)
plt.show()
第二个重点函数group_keypoints
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs)
pose_entries = []
# 将列表转成数组 [97,4]
# 将所有关节点展开成N*4的array
all_keypoints = np.array([item for sublist in all_keypoints_by_type for item in sublist])
points_per_limb = 10
grid = np.arange(points_per_limb, dtype=np.float32).reshape(1, -1, 1)
# 转成18x5
all_keypoints_by_type = [np.array(keypoints, np.float32) for keypoints in all_keypoints_by_type]
for part_id in range(len(BODY_PARTS_PAF_IDS)):
# part_pafs[128,136,2]
# 得到当前躯干的2维单位向量（yx）
part_pafs = pafs[:, :, BODY_PARTS_PAF_IDS[part_id]]
# 取出第part_id个关键点的[x,y,conf,id]
kpts_a = all_keypoints_by_type[BODY_PARTS_KPT_IDS[part_id][0]]# 当前躯干所有起点 BODY_PARTS_KPT_IDS为将关节点连接成躯干的映射
kpts_b = all_keypoints_by_type[BODY_PARTS_KPT_IDS[part_id][1]]# 当前躯干所有终点 kpts_a和kpts_b为[]，里面可能有几个4维向量，也可能为空
n = len(kpts_a)
m = len(kpts_b)
# 当前躯干无关节点
if n == 0 or m == 0:
continue
# Get vectors between all pairs of keypoints, i.e. candidate limb vectors.
#a是关键点起点，b是关键点终点
a = kpts_a[:, :2]
a = np.broadcast_to(a[None], (m, n, 2))
b = kpts_b[:, :2]
# 求b-a向量-->[m*n,1,2]
vec_raw = (b[:, None, :] - a).reshape(-1, 1, 2)
# print(vec_raw.shape)
# 产生一些点做积分
# Sample points along every candidate limb vector.
steps = (1 / (points_per_limb - 1) * vec_raw)
points = steps * grid + a.reshape(-1, 1, 2)
points = points.round().astype(dtype=np.int32)
x = points[..., 0].ravel()
y = points[..., 1].ravel()
# Compute affinity score between candidate limb vectors and part affinity field.
# 得到起点和终点中间抽点处paf的xy向量
field = part_pafs[y, x].reshape(-1, points_per_limb, 2)
# 求模
vec_norm = np.linalg.norm(vec_raw, ord=2, axis=-1, keepdims=True)
# 单位向量
vec = vec_raw / (vec_norm + 1e-6)
# 该向量在起点指向终点单位向量上的投影(不懂可看下面链接)
# https://www.bilibili.com/video/BV1Ca411u735?p=10&vd_source=b27506b0ebf497f4871f29b1dde01c74
affinity_scores = (field * vec).sum(-1).reshape(-1, points_per_limb)
# 投影大于阈值
valid_affinity_scores = affinity_scores > min_paf_score
# 累计valid插值点个数
valid_num = valid_affinity_scores.sum(1)
affinity_scores = (affinity_scores * valid_affinity_scores).sum(1) / (valid_num + 1e-6)
# 插值点中大于阈值的点的数量占总插值点数量的比例
success_ratio = valid_num / points_per_limb
# Get a list of limbs according to the obtained affinity score.
# 累计paf平均值大于0,且两关节点之间插值的点大于阈值的点的比例大于阈值
valid_limbs = np.where(np.logical_and(affinity_scores > 0, success_ratio > 0.8))[0]
if len(valid_limbs) == 0:
continue
b_idx, a_idx = np.divmod(valid_limbs, n)
affinity_scores = affinity_scores[valid_limbs]
# Suppress incompatible connections.
a_idx, b_idx, affinity_scores = connections_nms(a_idx, b_idx, affinity_scores)
# 当前躯干起点在所有关节点中的索引，终点在所有关节点中的索引，paf均值
connections = list(zip(kpts_a[a_idx, 3].astype(np.int32),
kpts_b[b_idx, 3].astype(np.int32),
affinity_scores))
if len(connections) == 0:# 当前无躯干，计算下一个躯干
continue
#下面整个流程看下方图片
#图片摘自链接https://www.cnblogs.com/darkknightzh/p/12152119.html#_lab2_3_9
# 第一次计算躯干
if part_id == 0:
# 前18个为每个人各个关节点在所有关节点中的索引，最后两个分别为总分值和分配给这个人关节点的数量
pose_entries = [np.ones(pose_entry_size) * -1 for _ in range(len(connections))]
for i in range(len(connections)):# 依次遍历当前找到的所有该躯干
pose_entries[i][BODY_PARTS_KPT_IDS[0][0]] = connections[i][0]# 起点在所有关节点中的索引
pose_entries[i][BODY_PARTS_KPT_IDS[0][1]] = connections[i][1]# 终点在所有关节点中的索引
pose_entries[i][-1] = 2# 当前人所有关节点的数量
pose_entries[i][-2] = np.sum(all_keypoints[connections[i][0:2], 2]) + connections[i][2] # 两个关节点热图值+平均paf值
elif part_id == 17 or part_id == 18:# 最后两个躯干
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
for i in range(len(connections)):# 将当前躯干和part_id=0时分配的所有人依次比较。此处为当前躯干
for j in range(len(pose_entries)):# 此处为分配的所有人
if pose_entries[j][kpt_a_id] == connections[i][0] and pose_entries[j][kpt_b_id] == -1:# 当前躯干的起点和分配到的某个人的起点一致，且当前躯干的终点未分配
pose_entries[j][kpt_b_id] = connections[i][1]# 将当前躯干的终点分配到这个人对应终点上
elif pose_entries[j][kpt_b_id] == connections[i][1] and pose_entries[j][kpt_a_id] == -1:# 当前躯干的终点和分配到的某个人的终点一致，且当前躯干的起点未分配
pose_entries[j][kpt_a_id] = connections[i][0]# 将当前躯干的起点分配到这个人对应起点上
continue
else:
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
for i in range(len(connections)):# 将当前躯干和part_id=0时分配的所有人依次比较。此处为当前躯干
num = 0
for j in range(len(pose_entries)): # 此处为分配的所有人
if pose_entries[j][kpt_a_id] == connections[i][0]:# 当前躯干的起点和分配到的某个人的起点一致
pose_entries[j][kpt_b_id] = connections[i][1]# 将当前躯干的终点分配到这个人对应终点上
num += 1# 分配的人+1
pose_entries[j][-1] += 1# 当前人所有关节点的数量+1
pose_entries[j][-2] += all_keypoints[connections[i][1], 2] + connections[i][2]# 当前人socre增加
if num == 0:# 如果没有分配到的人，则新建一个人
pose_entry = np.ones(pose_entry_size) * -1
pose_entry[kpt_a_id] = connections[i][0]
pose_entry[kpt_b_id] = connections[i][1]
pose_entry[-1] = 2
pose_entry[-2] = np.sum(all_keypoints[connections[i][0:2], 2]) + connections[i][2]
pose_entries.append(pose_entry)
filtered_entries = []# 依次遍历所有分配的人
for i in range(len(pose_entries)):
if pose_entries[i][-1] < 3 or (pose_entries[i][-2] / pose_entries[i][-1] < 0.2):# 如果当前人关节点数量少于3,或者当前人平均得分小于0.2,则删除该人
continue
filtered_entries.append(pose_entries[i])
pose_entries = np.asarray(filtered_entries)# 返回所有分配的人（前18维为每个人各个关节点在所有关节点中的索引，后两唯为每个人得分及每个人关节点数量），及所有关节点信息
return pose_entries, all_keypoints
流程图
匈牙利算法
匈牙利讲解1
匈牙利讲解2
# openpose的匈牙利算法
def connections_nms(a_idx, b_idx, affinity_scores):
# From all retrieved connections that share the same starting/ending keypoints leave only the top-scoring ones.
# 排序
order = affinity_scores.argsort()[::-1]
affinity_scores = affinity_scores[order]
# 对应Boys
a_idx = a_idx[order]
# 对应Girls
b_idx = b_idx[order]
idx = []
has_kpt_a = set()
has_kpt_b = set()
for t, (i, j) in enumerate(zip(a_idx, b_idx)):
# 起点和终点均未被占用(如果i某个起点或者某个终点被分配给了不同的躯干，因paf从大到小排序，故paf较小的忽略)
if i not in has_kpt_a and j not in has_kpt_b:
idx.append(t)
has_kpt_a.add(i)# 对应起点被占用
has_kpt_b.add(j)# 对应终点被占用
idx = np.asarray(idx, dtype=np.int32)
return a_idx[idx], b_idx[idx], affinity_scores[idx]