PV-RCNN代码解读——eval.py
PV-RCNN：paper，code
（一）目标检测模型评估的简介
训练完数据并且测试最后一个epoch之后，log的结果
INFO Car AP@0.70, 0.70, 0.70:
bbox AP:89.6559, 83.1009, 78.4558
bev AP:87.8147, 77.5440, 76.1072
3d AP:75.0818, 64.6598, 58.0468
aos AP:87.94, 80.87, 75.62
这里涉及到目标检测中几个重要的定义：
IoU（Intersection over union）：交并比IoU衡量的是两个区域的重叠程度，是两个区域重叠部分面积占二者总面积的比例。在目标检测中，如果模型输出的结果与真值gt的交并比 > 某个阈值（0.5或0.7）时，即认为我们的模型输出了正确的结果。
Precision ：检索出来的条目中有多大比例是我们需要的。
Recall：我们需要的条目中有多大比例被检索出来了。
AP(Average Precision)：平均精准度，对Precision-Recall曲线上的Precision值求均值。
（二）evaluation函数
当你运行test.py进行测试，会用到evaluation函数
# tools/test.py
result_str, result_dict = dataset.evaluation(
det_annos, class_names,
eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
output_path=final_output_dir
)
evaluation函数的定义
# pcdet/datasets/kitti/kitti_dataset.py
def evaluation(self, det_annos, class_names, **kwargs):
from .kitti_object_eval_python import eval as kitti_eval
eval_det_annos = copy.deepcopy(det_annos)
eval_gt_annos = [copy.deepcopy(info['annos']) for info in self.kitti_infos]
ap_result_str, ap_dict = kitti_eval.get_official_eval_result(eval_gt_annos, eval_det_annos, class_names)
return ap_result_str, ap_dict
get_official_eval_result函数的定义
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
# 打印测试结果
def get_official_eval_result(gt_annos, dt_annos, current_classes, PR_detail_dict=None):
overlap_0_7 = np.array([[0.7, 0.5, 0.5, 0.7, 0.5, 0.7],
[0.7, 0.5, 0.5, 0.7, 0.5, 0.7],
[0.7, 0.5, 0.5, 0.7, 0.5, 0.7]])
overlap_0_5 = np.array([[0.7, 0.5, 0.5, 0.7, 0.5, 0.5],
[0.5, 0.25, 0.25, 0.5, 0.25, 0.5],
[0.5, 0.25, 0.25, 0.5, 0.25, 0.5]])
# 目的就是给不同的类别设置不同的阈值
# 每个类别输出2*2个结果，AP或AP_R40，overlap=0.7或0.5
min_overlaps = np.stack([overlap_0_7, overlap_0_5], axis=0) # [2, 3, 5]
class_to_name = {
0: 'Car', #[[0.7,0.7,0.7],[0.5, 0.5, 0.5]]
1: 'Pedestrian', #[[0.5,0.5,0.5],[0.5,0.25,0.25]]
2: 'Cyclist', #[[0.5,0.5,0.5],[0.5,0.25,0.25]]
3: 'Van', #[[0.7,0.7,0.7],[0.7, 0.5, 0.5]]
4: 'Person_sitting',#[[0.5,0.5,0.5],[0.5,0.25,0.25]]
5: 'Truck' #[[0.7,0.7,0.7],[0.5, 0.5, 0.5]]
}
name_to_class = {
v: n for n, v in class_to_name.items()}
min_overlaps = min_overlaps[:, :, current_classes]
result = ''
# 计算结果的函数do_eval()（后面详细说）
mAPbbox, mAPbev, mAP3d, mAPaos, mAPbbox_R40, mAPbev_R40, mAP3d_R40, mAPaos_R40 = do_eval(
gt_annos, dt_annos, current_classes, min_overlaps, compute_aos, PR_detail_dict=PR_detail_dict)
#打印结果
ret_dict = {
}
for j, curcls in enumerate(current_classes): #每一种类
# mAP threshold array: [num_minoverlap, metric, class]
# mAP result: [num_class, num_diff, num_minoverlap]
for i in range(min_overlaps.shape[0]): # 2
result += print_str(
(f"{
class_to_name[curcls]} "
"AP@{:.2f}, {:.2f}, {:.2f}:".format(*min_overlaps[i, :, j])))
result += print_str((f"bbox AP:{
mAPbbox[j, 0, i]:.4f}, "
f"{
mAPbbox[j, 1, i]:.4f}, "
f"{
mAPbbox[j, 2, i]:.4f}"))
result += print_str((f"bev AP:{
mAPbev[j, 0, i]:.4f}, "
f"{
mAPbev[j, 1, i]:.4f}, "
f"{
mAPbev[j, 2, i]:.4f}"))
result += print_str((f"3d AP:{
mAP3d[j, 0, i]:.4f}, "
f"{
mAP3d[j, 1, i]:.4f}, "
f"{
mAP3d[j, 2, i]:.4f}"))
if compute_aos:
result += print_str((f"aos AP:{
mAPaos[j, 0, i]:.2f}, "
f"{
mAPaos[j, 1, i]:.2f}, "
f"{
mAPaos[j, 2, i]:.2f}"))
'''省略了把AP换成AP_R40的部分，因为代码是类似的'''
return result, ret_dict
用到的几个函数定义如下
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def print_str(value, *arg, sstream=None): #打印结果的函数
if sstream is None:
sstream = sysio.StringIO()
sstream.truncate(0)
sstream.seek(0)
print(value, *arg, file=sstream)
return sstream.getvalue()
（三）do_eval函数
do_eval是计算评估结果的重要函数，其定义如下
# 计算评估结果
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def do_eval(gt_annos, dt_annos, current_classes, min_overlaps, compute_aos=False, PR_detail_dict=None):
# min_overlaps: [num_minoverlap, metric, num_class]
difficultys = [0, 1, 2]
'''这里只保留了计算3d的代码，bev,aos,bbox是类似的'''
ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 2, min_overlaps)
# ret: [num_class, num_diff, num_minoverlap, num_sample_points]
mAP_3d = get_mAP(ret["precision"])
mAP_3d_R40 = get_mAP_R40(ret["precision"])
if PR_detail_dict is not None:
PR_detail_dict['3d'] = ret['precision']
return mAP_bbox, mAP_bev, mAP_3d, mAP_aos, mAP_bbox_R40, mAP_bev_R40, mAP_3d_R40, mAP_aos_R40
get_mAP和get_mAP_R40函数的定义
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def get_mAP(prec): # 计算mAP
sums = 0
for i in range(0, prec.shape[-1], 4):
sums = sums + prec[..., i]
return sums / 11 * 100
def get_mAP_R40(prec): #计算mAP_R40
sums = 0
for i in range(1, prec.shape[-1]):
sums = sums + prec[..., i]
return sums / 40 * 100
eval_class的定义
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def eval_class(gt_annos,
dt_annos,
current_classes,
difficultys, # tuple类型， (0, 1, 2)
metric, # 0 (bbox), 1 (bev)或 2 (3d)
min_overlaps, # (2, 3, num_classes) 其中:
# 2 表示阈值为中等或者容易
# 3 表示表示不同的指标 (bbox, bev, 3d),
# num_classes用于每个类的阈值
compute_aos=False,
num_parts=100):
"""这里是官方注释的各个参量的意义
Kitti eval. support 2d/bev/3d/aos eval. support 0.5:0.05:0.95 coco AP.
Args:
gt_annos: dict, must from get_label_annos() in kitti_common.py
dt_annos: dict, must from get_label_annos() in kitti_common.py
current_classes: list of int, 0: car, 1: pedestrian, 2: cyclist
difficultys: list of int. eval difficulty, 0: easy, 1: normal, 2: hard
metric: eval type. 0: bbox, 1: bev, 2: 3d
min_overlaps: float, min overlap. format: [num_overlap, metric, class].
num_parts: int. a parameter for fast calculate algorithm
Returns:
dict of recall, precision and aos
"""
assert len(gt_annos) == len(dt_annos)
num_examples = len(gt_annos)
split_parts = get_split_parts(num_examples, num_parts)
# 计算iou，calculate_iou_partly函数解读代码参见文末链接
rets = calculate_iou_partly(dt_annos, gt_annos, metric, num_parts)
(overlaps, parted_overlaps, total_dt_num, total_gt_num) = rets
N_SAMPLE_PTS = 41
num_minoverlap = len(min_overlaps)
num_class = len(current_classes)
num_difficulty = len(difficultys)
# 初始化precision,recall,aos
precision = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
recall = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
aos = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
# 每个类别
for m, current_class in enumerate(current_classes):
# 每个难度
for l, difficulty in enumerate(difficultys):
# _prepare_data函数解读代码参见文末链接
rets = _prepare_data(gt_annos, dt_annos, current_class, difficulty)
(gt_datas_list, dt_datas_list, ignored_gts, ignored_dets,
dontcares, total_dc_num, total_num_valid_gt) = rets
# 运行两次，首先进行中等难度的总体设置，然后进行简单设置。
for k, min_overlap in enumerate(min_overlaps[:, metric, m]):
thresholdss = []
# 循环浏览数据集中的图像。因此一次只显示一张图片。
for i in range(len(gt_annos)):
# compute_statistics_jit函数解读代码参见文末链接
rets = compute_statistics_jit(
overlaps[i], # 单个图像的iou值b/n gt和dt
gt_datas_list[i], # N x 5阵列
dt_datas_list[i], # N x 6阵列
ignored_gts[i], # 长度N数组，-1、0、1
ignored_dets[i], # 长度N数组，-1、0、1
dontcares[i], # 无关框数量x 4
metric, # 0, 1, 或 2 (bbox, bev, 3d)
min_overlap=min_overlap, # 浮动最小IOU阈值为正
thresh=0.0, # 忽略得分低于此值的dt。
compute_fp=False)
tp, fp, fn, similarity, thresholds, _ = rets
thresholdss += thresholds.tolist()
# 一维数组，记录匹配的dts分数
thresholdss = np.array(thresholdss)
thresholds = get_thresholds(thresholdss, total_num_valid_gt)
thresholds = np.array(thresholds)
# N_SAMPLE_PTS长度的一维数组，记录分数，递减，表示阈值
# 储存有关gt/dt框的信息（是否忽略，fn，tn，fp）
pr = np.zeros([len(thresholds), 4])
idx = 0
# 我们将数据集分成多个部分并运行。
for j, num_part in enumerate(split_parts):
gt_datas_part = np.concatenate(gt_datas_list[idx:idx + num_part], 0)
dt_datas_part = np.concatenate(dt_datas_list[idx:idx + num_part], 0)
dc_datas_part = np.concatenate(dontcares[idx:idx + num_part], 0)
ignored_dets_part = np.concatenate(ignored_dets[idx:idx + num_part], 0)
ignored_gts_part = np.concatenate(ignored_gts[idx:idx + num_part], 0)
# 再将各部分数据融合
fused_compute_statistics(parted_overlaps[j], pr, total_gt_num[idx:idx + num_part], total_dt_num[idx:idx + num_part], total_dc_num[idx:idx + num_part], gt_datas_part, dt_datas_part, dc_datas_part, ignored_gts_part, ignored_dets_part, metric, min_overlap=min_overlap, thresholds=thresholds, compute_aos=compute_aos)
idx += num_part
for i in range(len(thresholds)): #计算recall和precision
recall[m, l, k, i] = pr[i, 0] / (pr[i, 0] + pr[i, 2]) #! true pos / (true pos + false neg)
precision[m, l, k, i] = pr[i, 0] / (pr[i, 0] + pr[i, 1]) # true pos / (true pos + false pos)
if compute_aos:
aos[m, l, k, i] = pr[i, 3] / (pr[i, 0] + pr[i, 1])
# 返回各自序列的最值
for i in range(len(thresholds)):
precision[m, l, k, i] = np.max(precision[m, l, k, i:], axis=-1)
recall[m, l, k, i] = np.max(recall[m, l, k, i:], axis=-1)
if compute_aos:
aos[m, l, k, i] = np.max(aos[m, l, k, i:], axis=-1)
ret_dict = {
"recall": recall, # [num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS]
"precision": precision, # RECALLING RECALL的顺序，因此精度降低
"orientation": aos,
}
return ret_dict
致谢：detection-toolbox的作者将代码中的注释写的太清晰了，对我理解代码起到了很大帮助。有需要的读者可以看英文原版。
我的其他PV-RCNN代码解读系列文章，如果对你有帮助的话，请给我点赞哦~
PV-RCNN代码解读——输入参数介绍
PV-RCNN代码解读——TP，FP，TN，FN的计算
PV-RCNN代码解读——eval.py
PV-RCNN代码解读——计算iou
PV-RCNN代码解读——数据初始化
PV-RCNN代码解读——从点云到输入神经网络的数据处理