输出多标签分类模型每class指标OP,OR,OF1,CP,CR,CF1
需要逐类对预测结果进行分析。
相关代码地址：https://github.com/Xingxiangrui/multi_label_analyse
目录
一、结果读出
1.1 写出结果
1.2 结果读出
二、运算指标
2.1 指标与print
2.2 指标的运算
2.3 输出
三、precision，recall，F1
3.1 定义函数进行运算
3.2 函数的调用
3.3 运行结果
一、结果读出
1.1 写出结果
见 PyTorch实操(五)加载模型验证并将所有结果写入文件
1.2 结果读出
一定要确定路径之中为rb，不然会将文件重写写入wb，文件会被清零。
with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
model_results = pickle.load(f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
labels=pickle.load(f)
print('model_results[3]',model_results[3])
print('labels[3]',labels[3])二、运算指标
2.1 指标与print
需要算出OP，OR，OF1等各个指标。直接运用self.state['ap_meter']得出相应的指标
def on_end_epoch(self, training, model, criterion, data_loader, optimizer=None, display=True):
map = 100 * self.state['ap_meter'].value().mean()
loss = self.state['meter_loss'].value()[0]
OP, OR, OF1, CP, CR, CF1 = self.state['ap_meter'].overall()
OP_k, OR_k, OF1_k, CP_k, CR_k, CF1_k = self.state['ap_meter'].overall_topk(3)
if display:
if training:
print('Epoch: [{0}]\t'
'Loss {loss:.4f}\t'
'mAP {map:.3f}'.format(self.state['epoch'], loss=loss, map=map))
print('OP: {OP:.4f}\t'
'OR: {OR:.4f}\t'
'OF1: {OF1:.4f}\t'
'CP: {CP:.4f}\t'
'CR: {CR:.4f}\t'
'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
else:
print('Test: \t Loss {loss:.4f}\t mAP {map:.3f}'.format(loss=loss, map=map))
print('OP: {OP:.4f}\t'
'OR: {OR:.4f}\t'
'OF1: {OF1:.4f}\t'
'CP: {CP:.4f}\t'
'CR: {CR:.4f}\t'
'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
print('OP_3: {OP:.4f}\t'
'OR_3: {OR:.4f}\t'
'OF1_3: {OF1:.4f}\t'
'CP_3: {CP:.4f}\t'
'CR_3: {CR:.4f}\t'
'CF1_3: {CF1:.4f}'.format(OP=OP_k, OR=OR_k, OF1=OF1_k, CP=CP_k, CR=CR_k, CF1=CF1_k))
return map相应的函数为.overall()
2.2 指标的运算
def overall(self):
if self.scores.numel() == 0:
return 0
scores = self.scores.cpu().numpy()
targets = self.targets.cpu().numpy()
targets[targets == -1] = 0
return self.evaluation(scores, targets)
def overall_topk(self, k):
targets = self.targets.cpu().numpy()
targets[targets == -1] = 0
n, c = self.scores.size()
scores = np.zeros((n, c)) - 1
index = self.scores.topk(k, 1, True, True)[1].cpu().numpy()
tmp = self.scores.cpu().numpy()
for i in range(n):
for ind in index[i]:
scores[i, ind] = 1 if tmp[i, ind] >= 0 else -1
return self.evaluation(scores, targets)
def evaluation(self, scores_, targets_):
n, n_class = scores_.shape
Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
for k in range(n_class):
scores = scores_[:, k]
targets = targets_[:, k]
targets[targets == -1] = 0
Ng[k] = np.sum(targets == 1)
Np[k] = np.sum(scores >= 0)
Nc[k] = np.sum(targets * (scores >= 0))
Np[Np == 0] = 1
OP = np.sum(Nc) / np.sum(Np)
OR = np.sum(Nc) / np.sum(Ng)
OF1 = (2 * OP * OR) / (OP + OR)
CP = np.sum(Nc / Np) / n_class
CR = np.sum(Nc / Ng) / n_class
CF1 = (2 * CP * CR) / (CP + CR)
return OP, OR, OF1, CP, CR, CF12.3 输出
输入为 self.state['ap_meter']
其中包括了 self.scores和self.targets,且最终转为了numpy数组
scores = self.scores.cpu().numpy()
targets = self.targets.cpu().numpy()在往上，此变量是通过add加入了网络的预测输出，与label
def on_end_batch(self, training, model, criterion, data_loader, optimizer=None, display=True):
Engine.on_end_batch(self, training, model, criterion, data_loader, optimizer, display=False)
# measure mAP
self.state['ap_meter'].add(self.state['output'].data, self.state['target_gt'])三、precision，recall，F1
3.1 定义函数进行运算
我们自己定义函数运算相应的指标
def scores_evaluation(scores_, targets_):
print('evaluation start...')
n, n_class = scores_.shape
print('img_numbers=',n,'n_class=',n_class)
Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
cls_P,cls_R,cls_F1=np.zeros(n_class),np.zeros(n_class),np.zeros(n_class)
for k in range(n_class):
scores = scores_[:, k] # all img scores on class_k
targets = targets_[:, k] # all img labels on class_k
targets[targets == -1] = 0 # set img labels from -1 to 0
Ng[k] = np.sum(targets == 1) # ture: all ture labels sum number
Np[k] = np.sum(scores >= 0) # positive: all predict positive sum number
Nc[k] = np.sum(targets * (scores >= 0)) # true_positive: true_positive sum number
cls_P[k]=Nc[k]/Np[k]
cls_R[k]=Nc[k]/Ng[k]
cls_F1[k]=(2 * cls_P[k] * cls_R[k]) / (cls_P[k] + cls_R[k])
Np[Np == 0] = 1
print('np.sum(Nc),true_positive=',np.sum(Nc))
print('np.sum(Np),positive=',np.sum(Np))
print('np.sum(Ng),ture=',np.sum(Ng))
# for all labels num_imgs*n_classes
OP = np.sum(Nc) / np.sum(Np) # precision: true_positive/positive
OR = np.sum(Nc) / np.sum(Ng) # recall: true_positive/true
OF1 = (2 * OP * OR) / (OP + OR) # F1_score: harmonic mean of precision and recall
# average by class
CP = np.sum(Nc / Np) / n_class # precision: true_positive/positive
CR = np.sum(Nc / Ng) / n_class # recall: true_positive/true
CF1 = (2 * CP * CR) / (CP + CR) # F1_score: harmonic mean of precision and recall
return OP, OR, OF1, CP, CR, CF1,cls_P,cls_R,cls_F13.2 函数的调用
直接调用函数进行运算，加载出模型，需要为numpy格式，dim0为img_idx, dim1为classes，直接讲numpy送入函数即可returen出需要的值。
# load results on pkl files
# file in format { 0: [[ -4.905565 -7.9314375 ... -6.8639855 -8.622047 -8.28002]] numpy 1*80
# 1: [[ -8.905565 -8.9314375 ... -5.8639855 -6.622047 -4.28002]] numpy 1*80
# ...
# 1355: .......... numpy 1*80 }
# label in format (0: [[1. 0. 0. 0. 0. 0. 0. 0. ..... 1 ]] numpy 1*80
# 1: [[0. 1. ..... ...]]}
with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
model_results = pickle.load(f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
labels=pickle.load(f)
# print('model_results[3]',model_results[3])
# print('labels[3]',labels[3])
# concat all numpy
total_results=model_results[0]
total_labels=labels[0]
for img_idx in range(len(model_results)-1):
if img_idx%1000==0:
print(img_idx,'/',len(model_results))
total_results=np.append(total_results,model_results[img_idx+1],axis=0)
total_labels=np.append(total_labels,labels[img_idx+1],axis=0)
print('np.shape(total_results)',np.shape(total_results))
print('np.shape(total_labels)',np.shape(total_labels))
with open('checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl', 'wb') as f:
print("writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl")
pickle.dump(total_results, f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_numpy.pkl','wb') as f:
print("writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl")
pickle.dump(total_labels, f)
OP,OR,OF1,CP,CR,CF1,cls_P,cls_R,cls_F1=scores_evaluation(scores_=total_results,targets_=total_labels)
all_evaluate_results={'OP':OP,'OR':OR,'OF1':OF1,'CP':CP,'CR':CR,'CF1':CF1,'cls_P':cls_P,'cls_R':cls_R,'cls_F1':cls_F1}
with open('checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl','wb') as f:
print('writing checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl')
pickle.dump(all_evaluate_results, f)最终将模型写入文件。
3.3 运行结果
针对结果
np.shape(total_results) (40137, 80)
np.shape(total_labels) (40137, 80)
writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl
writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl
evaluation start...
img_numbers= 40137 n_class= 80
np.sum(Nc),true_positive= 93107.0
np.sum(Np),positive= 122286.0
np.sum(Ng),ture= 116592.0
class_Precison [0.96273115 0.7880597 0.76769912 0.86618005 0.93787336 0.83281005
0.91071429 0.6272687 0.7920434 0.74237508 0.83457249 0.826
0.76213592 0.5688509 0.82555781 0.94503171 0.88401697 0.89315353
0.9 0.80672269 0.96170213 0.93786982 0.99096386 0.98452381
0.45626478 0.83036406 0.45948866 0.80294358 0.70716889 0.83989835
0.86316776 0.7015625 0.71230398 0.91428571 0.82326112 0.8512931
0.89946619 0.90539484 0.96890756 0.6308514 0.71782763 0.63008487
0.59097222 0.53621561 0.51172414 0.61934082 0.79247911 0.62579281
0.66046512 0.73846154 0.77116402 0.71869328 0.66125761 0.8952734
0.72968198 0.68949343 0.65346734 0.66802089 0.61627201 0.79196217
0.64725882 0.88643791 0.77614679 0.82966102 0.6955414 0.66727773
0.73536585 0.59127907 0.57725948 0.69874477 0.2293578 0.74602304
0.58472086 0.57300144 0.83785617 0.69811321 0.58131488 0.8566879
0.42105263 0.60055096]
class_recall [0.9385227 0.71095153 0.83014354 0.87612797 0.93452381 0.78592593
0.91569087 0.68920233 0.83587786 0.79610299 0.75844595 0.70118846
0.60153257 0.6109128 0.72613738 0.90608108 0.82182774 0.86013986
0.90184049 0.86486486 0.94957983 0.92961877 0.97193501 0.97408716
0.52674672 0.7695621 0.56161972 0.76242236 0.66438356 0.87318362
0.92749245 0.8424015 0.81730104 0.92434663 0.90362954 0.93491124
0.92582418 0.89628483 0.96647108 0.68956044 0.69724771 0.70336491
0.7254902 0.67730496 0.62405383 0.71339174 0.78159341 0.60285132
0.69437653 0.76056338 0.87014925 0.68512111 0.72123894 0.86481647
0.78967495 0.75851393 0.73819255 0.7948895 0.65909091 0.77786378
0.76919192 0.91561181 0.80469245 0.79464286 0.81008902 0.69932757
0.804 0.6 0.7734375 0.84428716 0.33783784 0.86404066
0.7556962 0.654814 0.72476526 0.70916667 0.55629139 0.74309392
0.22857143 0.63929619]
class F1 score [0.9504728 0.74752242 0.79770115 0.87112561 0.93619559 0.80868902
0.9131958 0.65677868 0.81337047 0.76830087 0.79469027 0.75849403
0.67237687 0.58913204 0.77266255 0.9251466 0.85178876 0.87633588
0.90091931 0.83478261 0.95560254 0.93372607 0.9813572 0.97927768
0.48897897 0.79880775 0.50544662 0.7821585 0.68510889 0.85621762
0.89417476 0.7655584 0.76119884 0.91928865 0.86157518 0.89114495
0.91245487 0.9008168 0.96768779 0.65890074 0.70738802 0.66471133
0.65135859 0.59855845 0.56233422 0.66304769 0.78699862 0.61410788
0.67699642 0.74934952 0.81767181 0.70150576 0.68994709 0.87978142
0.75849403 0.72235872 0.69325088 0.72595396 0.63696266 0.78484967
0.70297715 0.90078871 0.79016189 0.81177446 0.74845785 0.68292683
0.76815287 0.59560761 0.66110184 0.76465201 0.27322404 0.80070651
0.65930425 0.61118203 0.77721838 0.70359653 0.56852792 0.79585799
0.2962963 0.61931818]
All evaluate results:
OP: 0.7614 OR: 0.7986 OF1: 0.7795 CP: 0.7446 CR: 0.7693 CF1: 0.7568