智能数字图像处理：图卷积SGN代码（pytorch）之get_raw_denoised_data.py解读
1.root_path = './'
raw_data_file = osp.join(root_path, 'raw_data', 'raw_skes_data.pkl')
save_path = osp.join(root_path, 'denoised_data')-》载入上一个文件处理后的.pkl文件路径
2.if not osp.exists(save_path):
os.mkdir(save_path)-》如果保存路径不存在则创建此路径
3.rgb_ske_path = osp.join(save_path, 'rgb+ske')
if not osp.exists(rgb_ske_path):
os.mkdir(rgb_ske_path)-》创建save_path路径下的rgb+ske文件夹
4.actors_info_dir = osp.join(save_path, 'actors_info')
if not osp.exists(actors_info_dir):
os.mkdir(actors_info_dir)-》创建save_path路径下的actors_info文件夹
5.missing_count = 0
noise_len_thres = 11
noise_spr_thres1 = 0.8
noise_spr_thres2 = 0.69754
noise_mot_thres_lo = 0.089925
noise_mot_thres_hi = 2-》创建各种参数
6.noise_len_logger = logging.getLogger('noise_length')-》创建日志
noise_len_logger.setLevel(logging.INFO)-》设置日志等级
noise_len_logger.addHandler(logging.FileHandler(osp.join(save_path, 'noise_length.log')))-》存储日志
noise_len_logger.info('{:^20}\t{:^17}\t{:^8}\t{}'.format('Skeleton', 'bodyID', 'Motion', 'Length'))-》日志信息格式
首先介绍get_raw_denoised_data方法
1.此函数功能：从原始骨架序列中获取去噪数据(关节位置和颜色位置)。
解释:对于骨骼序列的每一帧，都表示一个演员的25个关节的三维位置通过一个二维数组(形状:25 x 3)被重新塑造成一个75-维矢量通过连接每个3-维(x, y, z)坐标沿行维按关节顺序排列。每一帧包含两个演员的关节位置构成一个150-维向量。如果只有一个演员，然后最后75个值被0填充。否则，选择主参与者和第二个演员基于动作量。将每个150-维向量作为行向量放入一个二维numpy数组，其中行数等于有效帧数。所有这些2D数组被放到一个列表中，最后这个列表被序列化到一个cPickle文件中。对于包含两个或更多参与者的骨架序列(大多对应于最后11个类)，文件名和参与者的信息被记录到日志文件中。为了更好地理解，还可以生成可视化的RGB+骨架视频。
2.with open(raw_data_file, 'rb') as fr: # load raw skeletons data
raw_skes_data = pickle.load(fr)-》打开.pkl文件
3.num_skes = len(raw_skes_data)
print('Found %d available skeleton sequences.' % num_skes)-》打印文件有多少数据总量
3.raw_denoised_joints = []
raw_denoised_colors = []
frames_cnt = []-》创建3个空数组
4. for (idx, bodies_data) in enumerate(raw_skes_data):-》遍历.pkl文件取出id和骨骼名
ske_name = bodies_data['name']-》取出bodies_data中ID为name这一列的值
print('Processing %s' % ske_name)-》打印
num_bodies = len(bodies_data['data'])-》取出“”data“”列的数据长度
5.if num_bodies == 1: # only 1 actor-》如果帧中只有一个人物
num_frames = bodies_data['num_frames']-》获取此人的动作帧数
body_data = list(bodies_data['data'].values())[0]-》取出骨骼数据的第一行
joints, colors = get_one_actor_points(body_data, num_frames)-》获取关节和像素值
else: -》 超过一个演员，选择两个主要演员
joints, colors = get_two_actors_points(bodies_data)-》获取两个演员的关节
# Remove missing frames
joints, colors = remove_missing_frames(ske_name, joints, colors)-》移除掉有丢失的帧的关节像素。
num_frames = joints.shape[0] # 更新关节总数
6. raw_denoised_joints.append(joints)
raw_denoised_colors.append(colors)
frames_cnt.append(num_frames)-》把joints，colors，num_frames传入数组变量
7.if (idx + 1) % 1000 == 0:
print('Processed: %.2f%% (%d / %d), ' % \
(100.0 * (idx + 1) / num_skes, idx + 1, num_skes) + \
'Missing count: %d' % missing_count)-》每处理1000次打印一次处理信息
8.
raw_skes_joints_pkl = osp.join(save_path, 'raw_denoised_joints.pkl')
with open(raw_skes_joints_pkl, 'wb') as f:
pickle.dump(raw_denoised_joints, f, pickle.HIGHEST_PROTOCOL)
raw_skes_colors_pkl = osp.join(save_path, 'raw_denoised_colors.pkl')
with open(raw_skes_colors_pkl, 'wb') as f:
pickle.dump(raw_denoised_colors, f, pickle.HIGHEST_PROTOCOL)-》打开raw_denoised_joints.pkl和raw_denoised_colors.pkl文件
9.
frames_cnt = np.array(frames_cnt, dtype=np.int)
np.savetxt(osp.join(save_path, 'frames_cnt.txt'), frames_cnt, fmt='%d')
print('Saved raw denoised positions of {} frames into {}'.format(np.sum(frames_cnt),
raw_skes_joints_pkl))
print('Found %d files that have missing data' % missing_count)-》保存总帧数到指定路径的txt文件里面去。
解读get_two_actors_points方法：
作用：获得第一个和第二个演员的关节位置和颜色位置。
数据构成：bodies_data (dict): 3个键值对:'name'， 'data'， 'num_frames'。
bodies_data['data'] 也是一个字典, 当这个键是身体ID, 值是对应的身体数据（关节像素），它也是一个有4个键的字典:
关节:原始的三维关节位置。形状:(num_frames x 25,3)
颜色:原始的2D颜色位置。形状:(num_frames, 25,2)
间隔:记录帧索引的列表。
运动:运动数量
返回的是关节,颜色坐标。
1.ske_name = bodies_data['name']-》提取骨骼名
2.label = int(ske_name[-2:])-》取出所有name作为标签。
3.num_frames = bodies_data['num_frames']-》去帧总数
4.bodies_info = get_bodies_info(bodies_data['data'])-》取body数据
5. bodies_data, noise_info = denoising_bodies_data(bodies_data) -》把body数据放入去噪方法中处理
bodies_info += noise_info-》作body信息和噪声信息的和
6. bodies_data = list(bodies_data)-》放入列表
7.if len(bodies_data) == 1: -》判断body数据是不是一个人的
if label >= 50: -》双主体动作去噪失败
fail_logger_2.info(ske_name)-》记录日志
bodyID, body_data = bodies_data[0]-》取body数据第一个的ID和数据
joints, colors = get_one_actor_points(body_data, num_frames)-》将body数据和总帧数传入get_one_actor_points取出关节和像素
bodies_info += 'Main actor: %s' % bodyID-》将ID加入到bodies_info 中
else:
if label < 50: -》单主体动作去噪失败
fail_logger_1.info(ske_name)-》记录日志
8.joints = np.zeros((num_frames, 150), dtype=np.float32)-》建立一个行数为总帧数，150列的填充0的矩阵
9.colors = np.ones((num_frames, 2, 25, 2), dtype=np.float32) * np.nan-》建立一个带总帧数的以1为填充的4维矩阵。
10.bodyID, actor1 = bodies_data[0]-》取body数据第一个的ID和数据
11.start1, end1 = actor1['interval'][0], actor1['interval'][-1]-》记录bodies_data[0]行为数据的第一行和最后一行
12.joints[start1:end1 + 1, :75] = actor1['joints'].reshape(-1, 75)--》取出行为数据中的关节数据
13.colors[start1:end1 + 1, 0] = actor1['colors']--》取出行为数据中的像素数据
14.actor1_info = '{:^17}\t{}\t{:^8}\n'.format('Actor1', 'Interval', 'Motion') + \
'{}\t{:^8}\t{:f}\n'.format(bodyID, str([start1, end1]), actor1['motion'])-》匹配正则表达式的字符串
15. del bodies_data[0]-》删除bodies_data[0]对象
16.actor2_info = '{:^17}\t{}\t{:^8}\n'.format('Actor2', 'Interval', 'Motion')-》-》匹配正则表达式的字符串
17.start2, end2 = [0, 0] -》actor2的初始间隔(虚拟)
18.while len(bodies_data) > 0:-》如果数据不为空
bodyID, actor = bodies_data[0]-》取body数据第一个的ID和数据
start, end = actor['interval'][0], actor['interval'][-1]-》记录actor中间隔数据的第一行和最后一行
19.if min(end1, end) - max(start1, start) <= 0: -》与actor1没有重叠
joints[start:end + 1, :75] = actor['joints'].reshape(-1, 75)-》取出关节数据并重新定形
colors[start:end + 1, 0] = actor['colors']-》取出像素数据
actor1_info += '{}\t{:^8}\t{:f}\n'.format(bodyID, str([start, end]), actor['motion']) -》 更新actor1的间隔
start1 = min(start, start1)-》更新start1值
end1 = max(end, end1)-》更新end1值
20.del bodies_data[0]-》删除bodies_data值
21.bodies_info += ('\n' + actor1_info + '\n' + actor2_info)-》更新body信息，加入actor1_info和actor2_info信息
22. with open(osp.join(actors_info_dir, ske_name + '.txt'), 'w') as fw:
fw.write(bodies_info + '\n')-》写入txt文件
解读get_bodies_info方法：
1.bodies_info = '{:^17}\t{}\t{:^8}\n'.format('bodyID', 'Interval', 'Motion')-》规定bodies_info格式
2.for (bodyID, body_data) in bodies_data.items():-》循环遍历出bodyID, body_data
start, end = body_data['interval'][0], body_data['interval'][-1]-》取出第一个和最后一个索引
bodies_info += '{}\t{:^8}\t{:f}\n'.format(bodyID, str([start, end]), body_data['motion'])-》添加到bodies_info
3.return bodies_info + '\n'-》返回bodies_info信息
解读remove_missing_frames方法：
作用：切掉所有关节位置为0的缺失帧 。
对于有2个演员数据的序列，也记录缺失帧数actor1和actor2(用于调试)。
1.num_frames = joints.shape[0]-》总帧数
num_bodies = colors.shape[1] -》主体数
2.if num_bodies == 2: -》如果主体数为2
missing_indices_1 = np.where(joints[:, :75].sum(axis=1) == 0)[0]-》查询主体1丢失的索引
missing_indices_2 = np.where(joints[:, 75:].sum(axis=1) == 0)[0]-》查询主体2丢失的索引
cnt1 = len(missing_indices_1)-》记录主体1丢失的索引的长度
cnt2 = len(missing_indices_2)-》记录主体2丢失的索引的长度
3.start = 1 if 0 in missing_indices_1 else 0-》判断start 值
end = 1 if num_frames - 1 in missing_indices_1 else 0-》判断end的值
4.if max(cnt1, cnt2) > 0:-》写入日志
if cnt1 > cnt2:
info = '{}\t{:^10d}\t{:^6d}\t{:^6d}\t{:^5d}\t{:^3d}'.format(ske_name, num_frames,
cnt1, cnt2, start, end)
missing_skes_logger1.info(info)
else:
info = '{}\t{:^10d}\t{:^6d}\t{:^6d}'.format(ske_name, num_frames, cnt1, cnt2)
missing_skes_logger2.info(info)
找到数据没有丢失或丢失的有效框架索引
对于双主体动作，这意味着角色1和角色2的数据都缺失了。
5.valid_indices = np.where(joints.sum(axis=1) != 0)[0]-》基于指数
missing_indices = np.where(joints.sum(axis=1) == 0)[0]-》查询索引
num_missing = len(missing_indices)-》记录长度
6.if num_missing > 0: -》更新关节和颜色
joints = joints[valid_indices]
colors[missing_indices] = np.nan
global missing_count
missing_count += 1
missing_skes_logger.info('{}\t{:^10d}\t{:^11d}'.format(ske_name, num_frames, num_missing))-》写入日志
解读get_one_actor_points方法：
作用：只为一个演员找关节和颜色。
对于关节，每个坐标系包含75个X-Y-Z坐标。
对于颜色，每个帧包含25 x 2 (x, Y)坐标。
1.joints = np.zeros((num_frames, 75), dtype=np.float32)-》创建关节数组
2.colors = np.ones((num_frames, 1, 25, 2), dtype=np.float32) * np.nan-》创建像素数组
3. start, end = body_data['interval'][0], body_data['interval'][-1]-》记录start, end值
4. joints[start:end + 1] = body_data['joints'].reshape(-1, 75)-》载入关节值
5. colors[start:end + 1, 0] = body_data['colors']-》载入像素值
解读denoising_bodies_data方法
作用：基于一些启发式方法去噪数据，不一定对所有样本都正确。
返回:元组:(bodyID, body_data)。
1.ske_name = bodies_data['name']-》取出name值
2.bodies_data = bodies_data['data']-》取出data值
3.bodies_data, noise_info_len = denoising_by_length(ske_name, bodies_data)-》步骤1:基于帧长去噪。
4.if len(bodies_data) == 1: -》步骤1后只剩下一个bodyID
return bodies_data.items(), noise_info_len-》返回可遍历的(键, 值) 元组数组bodies_data和长度。
基于扩散的去噪：
5.bodies_data, noise_info_spr, denoised_by_spr = denoising_by_spread(ske_name, bodies_data)-》获得body数据、噪声数据、去燥数据
6. if len(bodies_data) == 1:-》如果bodies_data长度为1
return bodies_data.items(), noise_info_len + noise_info_spr-》返回可遍历的(键, 值) 元组数组bodies_data和噪声数据噪声长度。
7.bodies_motion = dict() -》让身体运动
8.for (bodyID, body_data) in bodies_data.items():-》取出bodyID和body数据
bodies_motion[bodyID] = body_data['motion']-》取出对于ID的运动信息
9.bodies_motion = sorted(bodies_motion.items(), key=lambda x: x[1], reverse=True)-》基于运动对主体进行排序
denoised_bodies_data = list()-》创建一个空列表
10.for (bodyID, _) in bodies_motion:-》遍历运动信息
denoised_bodies_data.append((bodyID, bodies_data[bodyID]))-》将ID和ID对应bodies_data加入在去噪数据中
return denoised_bodies_data, noise_info_len + noise_info_spr-》返回去燥数据，噪声信息
解读denoising_by_motion方法：
作用过滤出运动超出预定义间隔范围的bodyID
1.bodies_motion = sorted(bodies_motion.items(), key=lambda x: x[1], reverse=True)-》根据动作对主体进行排序，返回元组列表
2.denoised_bodies_data = [(bodies_motion[0][0], bodies_data[bodies_motion[0][0]])]-》保留最大动作的身体数据
noise_info = str() -》将对象转化为适于人阅读的形式。
3. for (bodyID, motion) in bodies_motion[1:]:-》遍历取出bodyID和动作信息
if (motion < noise_mot_thres_lo) or (motion > noise_mot_thres_hi):-》如果motion 不正常的话
noise_info += 'Filter out: %s, %.6f (motion).\n' % (bodyID, motion)-》加入噪声信息
noise_mot_logger.info('{}\t{}\t{:.6f}'.format(ske_name, bodyID, motion))-》加入日志
else:
denoised_bodies_data.append((bodyID, bodies_data[bodyID]))-》否则正常去燥body数据
解读denoising_by_spread方法
作用：根据Y值和X值的扩散对数据进行去噪。滤除噪声帧比高于预先定义的bodyID。
1.noise_info = str()
2.denoised_by_spr = False-》标记该序列是否已被扩展处理。
3.new_bodies_data = bodies_data.copy()-》复制bodies_data数据
4.for (bodyID, body_data) in new_bodies_data.items():-》遍历得到bodyID, body_data
if len(bodies_data) == 1:-》若只有一条数据跳出
break
5.valid_frames = get_valid_frames_by_spread(body_data['joints'].reshape(-1, 25, 3))-》得到正常帧
6. num_frames = len(body_data['interval'])-》得到总帧数
7. num_noise = num_frames - len(valid_frames)-》得到总噪声数
8. if num_noise == 0:-》如果没有噪声，继续执行
continue
9.ratio = num_noise / float(num_frames)-》得到噪声率
10.motion = body_data['motion']-》取出body_data里面的动作数据
11.if ratio >= noise_spr_thres2: -》如果噪声率大于0.69754
del bodies_data[bodyID]-》删除bodies_data[bodyID]
denoised_by_spr = True-》更新denoised_by_spr 为true
noise_info += 'Filter out: %s (spread rate >= %.2f).\n' % (bodyID, noise_spr_thres2)-》添加噪声信息
noise_spr_logger.info('%s\t%s\t%.6f\t%.6f' % (ske_name, bodyID, motion, ratio))-》添加噪声日志
12.else: -》否则更新动作
joints = body_data['joints'].reshape(-1, 25, 3)[valid_frames]-》将关节数据重新组织赋值给joint
body_data['motion'] = min(motion, np.sum(np.var(joints.reshape(-1, 3), axis=0)))-》求关节的方差的和与动作值的最小值赋值给body_data['motion']
noise_info += '%s: motion %.6f -> %.6f\n' % (bodyID, motion, body_data['motion'])-》更新noise_info
解读get_valid_frames_by_spread方法
作用：根据X和Y的扩展找到有效的(或合理的)帧(索引)。
1.num_frames = points.shape[0]-》取出总帧数
valid_frames = []
2.for i in range(num_frames):-》遍历
x = points[i, :, 0]
y = points[i, :, 1]
3. if (x.max() - x.min()) <= noise_spr_thres1 * (y.max() - y.min()): -》若x小于等于0.8y
valid_frames.append(i)-》列表末尾添加新的对象i。
return valid_frames
解读denoising_by_length方法：
作用：根据每个bodyID的帧长度去噪数据。过滤长度小于或等于预定义阈值的bodyID
1.noise_info = str()
new_bodies_data = bodies_data.copy()-》复制数据
2.for (bodyID, body_data) in new_bodies_data.items():
length = len(body_data['interval'])
if length <= noise_len_thres:-》若长度小于11
noise_info += 'Filter out: %s, %d (length).\n' % (bodyID, length)-》添加噪声信息
noise_len_logger.info('{}\t{}\t{:.6f}\t{:^6d}'.format(ske_name, bodyID,-》写入日志
body_data['motion'], length))
del bodies_data[bodyID]-》删除对应数据
附上完整代码：
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import os
import os.path as osp
import numpy as np
import pickle
import logging
root_path = './'
raw_data_file = osp.join(root_path, 'raw_data', 'raw_skes_data.pkl')
save_path = osp.join(root_path, 'denoised_data')
if not osp.exists(save_path):
os.mkdir(save_path)
rgb_ske_path = osp.join(save_path, 'rgb+ske')
if not osp.exists(rgb_ske_path):
os.mkdir(rgb_ske_path)
actors_info_dir = osp.join(save_path, 'actors_info')
if not osp.exists(actors_info_dir):
os.mkdir(actors_info_dir)
missing_count = 0
noise_len_thres = 11
noise_spr_thres1 = 0.8
noise_spr_thres2 = 0.69754
noise_mot_thres_lo = 0.089925
noise_mot_thres_hi = 2
noise_len_logger = logging.getLogger('noise_length')
noise_len_logger.setLevel(logging.INFO)
noise_len_logger.addHandler(logging.FileHandler(osp.join(save_path, 'noise_length.log')))
noise_len_logger.info('{:^20}\t{:^17}\t{:^8}\t{}'.format('Skeleton', 'bodyID', 'Motion', 'Length'))
noise_spr_logger = logging.getLogger('noise_spread')
noise_spr_logger.setLevel(logging.INFO)
noise_spr_logger.addHandler(logging.FileHandler(osp.join(save_path, 'noise_spread.log')))
noise_spr_logger.info('{:^20}\t{:^17}\t{:^8}\t{:^8}'.format('Skeleton', 'bodyID', 'Motion', 'Rate'))
noise_mot_logger = logging.getLogger('noise_motion')
noise_mot_logger.setLevel(logging.INFO)
noise_mot_logger.addHandler(logging.FileHandler(osp.join(save_path, 'noise_motion.log')))
noise_mot_logger.info('{:^20}\t{:^17}\t{:^8}'.format('Skeleton', 'bodyID', 'Motion'))
fail_logger_1 = logging.getLogger('noise_outliers_1')
fail_logger_1.setLevel(logging.INFO)
fail_logger_1.addHandler(logging.FileHandler(osp.join(save_path, 'denoised_failed_1.log')))
fail_logger_2 = logging.getLogger('noise_outliers_2')
fail_logger_2.setLevel(logging.INFO)
fail_logger_2.addHandler(logging.FileHandler(osp.join(save_path, 'denoised_failed_2.log')))
missing_skes_logger = logging.getLogger('missing_frames')
missing_skes_logger.setLevel(logging.INFO)
missing_skes_logger.addHandler(logging.FileHandler(osp.join(save_path, 'missing_skes.log')))
missing_skes_logger.info('{:^20}\t{}\t{}'.format('Skeleton', 'num_frames', 'num_missing'))
missing_skes_logger1 = logging.getLogger('missing_frames_1')
missing_skes_logger1.setLevel(logging.INFO)
missing_skes_logger1.addHandler(logging.FileHandler(osp.join(save_path, 'missing_skes_1.log')))
missing_skes_logger1.info('{:^20}\t{}\t{}\t{}\t{}\t{}'.format('Skeleton', 'num_frames', 'Actor1',
'Actor2', 'Start', 'End'))
missing_skes_logger2 = logging.getLogger('missing_frames_2')
missing_skes_logger2.setLevel(logging.INFO)
missing_skes_logger2.addHandler(logging.FileHandler(osp.join(save_path, 'missing_skes_2.log')))
missing_skes_logger2.info('{:^20}\t{}\t{}\t{}'.format('Skeleton', 'num_frames', 'Actor1', 'Actor2'))
def denoising_by_length(ske_name, bodies_data):
"""
Denoising data based on the frame length for each bodyID.
Filter out the bodyID which length is less or equal than the predefined threshold.
"""
noise_info = str()
new_bodies_data = bodies_data.copy()
for (bodyID, body_data) in new_bodies_data.items():
length = len(body_data['interval'])
if length <= noise_len_thres:
noise_info += 'Filter out: %s, %d (length).\n' % (bodyID, length)
noise_len_logger.info('{}\t{}\t{:.6f}\t{:^6d}'.format(ske_name, bodyID,
body_data['motion'], length))
del bodies_data[bodyID]
if noise_info != '':
noise_info += '\n'
return bodies_data, noise_info
def get_valid_frames_by_spread(points):
"""
Find the valid (or reasonable) frames (index) based on the spread of X and Y.
:param points: joints or colors
"""
num_frames = points.shape[0]
valid_frames = []
for i in range(num_frames):
x = points[i, :, 0]
y = points[i, :, 1]
if (x.max() - x.min()) <= noise_spr_thres1 * (y.max() - y.min()): # 0.8
valid_frames.append(i)
return valid_frames
def denoising_by_spread(ske_name, bodies_data):
"""
Denoising data based on the spread of Y value and X value.
Filter out the bodyID which the ratio of noisy frames is higher than the predefined
threshold.
bodies_data: contains at least 2 bodyIDs
"""
noise_info = str()
denoised_by_spr = False # mark if this sequence has been processed by spread.
new_bodies_data = bodies_data.copy()
# for (bodyID, body_data) in bodies_data.items():
for (bodyID, body_data) in new_bodies_data.items():
if len(bodies_data) == 1:
break
valid_frames = get_valid_frames_by_spread(body_data['joints'].reshape(-1, 25, 3))
num_frames = len(body_data['interval'])
num_noise = num_frames - len(valid_frames)
if num_noise == 0:
continue
ratio = num_noise / float(num_frames)
motion = body_data['motion']
if ratio >= noise_spr_thres2: # 0.69754
del bodies_data[bodyID]
denoised_by_spr = True
noise_info += 'Filter out: %s (spread rate >= %.2f).\n' % (bodyID, noise_spr_thres2)
noise_spr_logger.info('%s\t%s\t%.6f\t%.6f' % (ske_name, bodyID, motion, ratio))
else: # Update motion
joints = body_data['joints'].reshape(-1, 25, 3)[valid_frames]
body_data['motion'] = min(motion, np.sum(np.var(joints.reshape(-1, 3), axis=0)))
noise_info += '%s: motion %.6f -> %.6f\n' % (bodyID, motion, body_data['motion'])
# TODO: Consider removing noisy frames for each bodyID
if noise_info != '':
noise_info += '\n'
return bodies_data, noise_info, denoised_by_spr
def denoising_by_motion(ske_name, bodies_data, bodies_motion):
"""
Filter out the bodyID which motion is out of the range of predefined interval
"""
# Sort bodies based on the motion, return a list of tuples
# bodies_motion = sorted(bodies_motion.items(), key=lambda x, y: cmp(x[1], y[1]), reverse=True)
bodies_motion = sorted(bodies_motion.items(), key=lambda x: x[1], reverse=True)
# Reserve the body data with the largest motion
denoised_bodies_data = [(bodies_motion[0][0], bodies_data[bodies_motion[0][0]])]
noise_info = str()
for (bodyID, motion) in bodies_motion[1:]:
if (motion < noise_mot_thres_lo) or (motion > noise_mot_thres_hi):
noise_info += 'Filter out: %s, %.6f (motion).\n' % (bodyID, motion)
noise_mot_logger.info('{}\t{}\t{:.6f}'.format(ske_name, bodyID, motion))
else:
denoised_bodies_data.append((bodyID, bodies_data[bodyID]))
if noise_info != '':
noise_info += '\n'
return denoised_bodies_data, noise_info
def denoising_bodies_data(bodies_data):
"""
Denoising data based on some heuristic methods, not necessarily correct for all samples.
Return:
denoised_bodies_data (list): tuple: (bodyID, body_data).
"""
ske_name = bodies_data['name']
bodies_data = bodies_data['data']
# Step 1: Denoising based on frame length.
bodies_data, noise_info_len = denoising_by_length(ske_name, bodies_data)
if len(bodies_data) == 1: # only has one bodyID left after step 1
return bodies_data.items(), noise_info_len
# Step 2: Denoising based on spread.
bodies_data, noise_info_spr, denoised_by_spr = denoising_by_spread(ske_name, bodies_data)
if len(bodies_data) == 1:
return bodies_data.items(), noise_info_len + noise_info_spr
bodies_motion = dict() # get body motion
for (bodyID, body_data) in bodies_data.items():
bodies_motion[bodyID] = body_data['motion']
# Sort bodies based on the motion
# bodies_motion = sorted(bodies_motion.items(), key=lambda x, y: cmp(x[1], y[1]), reverse=True)
bodies_motion = sorted(bodies_motion.items(), key=lambda x: x[1], reverse=True)
denoised_bodies_data = list()
for (bodyID, _) in bodies_motion:
denoised_bodies_data.append((bodyID, bodies_data[bodyID]))
return denoised_bodies_data, noise_info_len + noise_info_spr
# TODO: Consider denoising further by integrating motion method
# if denoised_by_spr: # this sequence has been denoised by spread
# bodies_motion = sorted(bodies_motion.items(), lambda x, y: cmp(x[1], y[1]), reverse=True)
# denoised_bodies_data = list()
# for (bodyID, _) in bodies_motion:
# denoised_bodies_data.append((bodyID, bodies_data[bodyID]))
# return denoised_bodies_data, noise_info
# Step 3: Denoising based on motion
# bodies_data, noise_info = denoising_by_motion(ske_name, bodies_data, bodies_motion)
# return bodies_data, noise_info
def get_one_actor_points(body_data, num_frames):
"""
Get joints and colors for only one actor.
For joints, each frame contains 75 X-Y-Z coordinates.
For colors, each frame contains 25 x 2 (X, Y) coordinates.
"""
joints = np.zeros((num_frames, 75), dtype=np.float32)
colors = np.ones((num_frames, 1, 25, 2), dtype=np.float32) * np.nan
start, end = body_data['interval'][0], body_data['interval'][-1]
joints[start:end + 1] = body_data['joints'].reshape(-1, 75)
colors[start:end + 1, 0] = body_data['colors']
return joints, colors
def remove_missing_frames(ske_name, joints, colors):
"""
Cut off missing frames which all joints positions are 0s
For the sequence with 2 actors' data, also record the number of missing frames for
actor1 and actor2, respectively (for debug).
"""
num_frames = joints.shape[0]
num_bodies = colors.shape[1] # 1 or 2
if num_bodies == 2: # DEBUG
missing_indices_1 = np.where(joints[:, :75].sum(axis=1) == 0)[0]
missing_indices_2 = np.where(joints[:, 75:].sum(axis=1) == 0)[0]
cnt1 = len(missing_indices_1)
cnt2 = len(missing_indices_2)
start = 1 if 0 in missing_indices_1 else 0
end = 1 if num_frames - 1 in missing_indices_1 else 0
if max(cnt1, cnt2) > 0:
if cnt1 > cnt2:
info = '{}\t{:^10d}\t{:^6d}\t{:^6d}\t{:^5d}\t{:^3d}'.format(ske_name, num_frames,
cnt1, cnt2, start, end)
missing_skes_logger1.info(info)
else:
info = '{}\t{:^10d}\t{:^6d}\t{:^6d}'.format(ske_name, num_frames, cnt1, cnt2)
missing_skes_logger2.info(info)
# Find valid frame indices that the data is not missing or lost
# For two-subjects action, this means both data of actor1 and actor2 is missing.
valid_indices = np.where(joints.sum(axis=1) != 0)[0] # 0-based index
missing_indices = np.where(joints.sum(axis=1) == 0)[0]
num_missing = len(missing_indices)
if num_missing > 0: # Update joints and colors
joints = joints[valid_indices]
colors[missing_indices] = np.nan
global missing_count
missing_count += 1
missing_skes_logger.info('{}\t{:^10d}\t{:^11d}'.format(ske_name, num_frames, num_missing))
return joints, colors
def get_bodies_info(bodies_data):
bodies_info = '{:^17}\t{}\t{:^8}\n'.format('bodyID', 'Interval', 'Motion')
for (bodyID, body_data) in bodies_data.items():
start, end = body_data['interval'][0], body_data['interval'][-1]
bodies_info += '{}\t{:^8}\t{:f}\n'.format(bodyID, str([start, end]), body_data['motion'])
return bodies_info + '\n'
def get_two_actors_points(bodies_data):
"""
Get the first and second actor's joints positions and colors locations.
# Arguments:
bodies_data (dict): 3 key-value pairs: 'name', 'data', 'num_frames'.
bodies_data['data'] is also a dict, while the key is bodyID, the value is
the corresponding body_data which is also a dict with 4 keys:
- joints: raw 3D joints positions. Shape: (num_frames x 25, 3)
- colors: raw 2D color locations. Shape: (num_frames, 25, 2)
- interval: a list which records the frame indices.
- motion: motion amount
# Return:
joints, colors.
"""
ske_name = bodies_data['name']
label = int(ske_name[-2:])
num_frames = bodies_data['num_frames']
bodies_info = get_bodies_info(bodies_data['data'])
bodies_data, noise_info = denoising_bodies_data(bodies_data) # Denoising data
bodies_info += noise_info
bodies_data = list(bodies_data)
if len(bodies_data) == 1: # Only left one actor after denoising
if label >= 50: # DEBUG: Denoising failed for two-subjects action
fail_logger_2.info(ske_name)
bodyID, body_data = bodies_data[0]
joints, colors = get_one_actor_points(body_data, num_frames)
bodies_info += 'Main actor: %s' % bodyID
else:
if label < 50: # DEBUG: Denoising failed for one-subject action
fail_logger_1.info(ske_name)
joints = np.zeros((num_frames, 150), dtype=np.float32)
colors = np.ones((num_frames, 2, 25, 2), dtype=np.float32) * np.nan
bodyID, actor1 = bodies_data[0] # the 1st actor with largest motion
start1, end1 = actor1['interval'][0], actor1['interval'][-1]
joints[start1:end1 + 1, :75] = actor1['joints'].reshape(-1, 75)
colors[start1:end1 + 1, 0] = actor1['colors']
actor1_info = '{:^17}\t{}\t{:^8}\n'.format('Actor1', 'Interval', 'Motion') + \
'{}\t{:^8}\t{:f}\n'.format(bodyID, str([start1, end1]), actor1['motion'])
del bodies_data[0]
actor2_info = '{:^17}\t{}\t{:^8}\n'.format('Actor2', 'Interval', 'Motion')
start2, end2 = [0, 0] # initial interval for actor2 (virtual)
while len(bodies_data) > 0:
bodyID, actor = bodies_data[0]
start, end = actor['interval'][0], actor['interval'][-1]
if min(end1, end) - max(start1, start) <= 0: # no overlap with actor1
joints[start:end + 1, :75] = actor['joints'].reshape(-1, 75)
colors[start:end + 1, 0] = actor['colors']
actor1_info += '{}\t{:^8}\t{:f}\n'.format(bodyID, str([start, end]), actor['motion'])
# Update the interval of actor1
start1 = min(start, start1)
end1 = max(end, end1)
elif min(end2, end) - max(start2, start) <= 0: # no overlap with actor2
joints[start:end + 1, 75:] = actor['joints'].reshape(-1, 75)
colors[start:end + 1, 1] = actor['colors']
actor2_info += '{}\t{:^8}\t{:f}\n'.format(bodyID, str([start, end]), actor['motion'])
# Update the interval of actor2
start2 = min(start, start2)
end2 = max(end, end2)
del bodies_data[0]
bodies_info += ('\n' + actor1_info + '\n' + actor2_info)
with open(osp.join(actors_info_dir, ske_name + '.txt'), 'w') as fw:
fw.write(bodies_info + '\n')
return joints, colors
def get_raw_denoised_data():
"""
Get denoised data (joints positions and color locations) from raw skeleton sequences.
For each frame of a skeleton sequence, an actor's 3D positions of 25 joints represented
by an 2D array (shape: 25 x 3) is reshaped into a 75-dim vector by concatenating each
3-dim (x, y, z) coordinates along the row dimension in joint order. Each frame contains
two actor's joints positions constituting a 150-dim vector. If there is only one actor,
then the last 75 values are filled with zeros. Otherwise, select the main actor and the
second actor based on the motion amount. Each 150-dim vector as a row vector is put into
a 2D numpy array where the number of rows equals the number of valid frames. All such
2D arrays are put into a list and finally the list is serialized into a cPickle file.
For the skeleton sequence which contains two or more actors (mostly corresponds to the
last 11 classes), the filename and actors' information are recorded into log files.
For better understanding, also generate RGB+skeleton videos for visualization.
"""
with open(raw_data_file, 'rb') as fr: # load raw skeletons data
raw_skes_data = pickle.load(fr)
num_skes = len(raw_skes_data)
print('Found %d available skeleton sequences.' % num_skes)
raw_denoised_joints = []
raw_denoised_colors = []
frames_cnt = []
for (idx, bodies_data) in enumerate(raw_skes_data):
ske_name = bodies_data['name']
print('Processing %s' % ske_name)
num_bodies = len(bodies_data['data'])
if num_bodies == 1: # only 1 actor
num_frames = bodies_data['num_frames']
body_data = list(bodies_data['data'].values())[0]
joints, colors = get_one_actor_points(body_data, num_frames)
else: # more than 1 actor, select two main actors
joints, colors = get_two_actors_points(bodies_data)
# Remove missing frames
joints, colors = remove_missing_frames(ske_name, joints, colors)
num_frames = joints.shape[0] # Update
# Visualize selected actors' skeletons on RGB videos.
raw_denoised_joints.append(joints)
raw_denoised_colors.append(colors)
frames_cnt.append(num_frames)
if (idx + 1) % 1000 == 0:
print('Processed: %.2f%% (%d / %d), ' % \
(100.0 * (idx + 1) / num_skes, idx + 1, num_skes) + \
'Missing count: %d' % missing_count)
raw_skes_joints_pkl = osp.join(save_path, 'raw_denoised_joints.pkl')
with open(raw_skes_joints_pkl, 'wb') as f:
pickle.dump(raw_denoised_joints, f, pickle.HIGHEST_PROTOCOL)
raw_skes_colors_pkl = osp.join(save_path, 'raw_denoised_colors.pkl')
with open(raw_skes_colors_pkl, 'wb') as f:
pickle.dump(raw_denoised_colors, f, pickle.HIGHEST_PROTOCOL)
frames_cnt = np.array(frames_cnt, dtype=np.int)
np.savetxt(osp.join(save_path, 'frames_cnt.txt'), frames_cnt, fmt='%d')
print('Saved raw denoised positions of {} frames into {}'.format(np.sum(frames_cnt),
raw_skes_joints_pkl))
print('Found %d files that have missing data' % missing_count)
if __name__ == '__main__':
get_raw_denoised_data()
https://github.com/microsoft/SGN