小黑啃fastNLP:自定义metrics解决分类问题
1.继承ClassfierMetric
from fastNLP.core.metrics import MetricBase
import numpy as np
from sklearn.metrics import f1_score, precision_score, recall_score, classification_report
class ClassfierMetric(MetricBase):
def __init__(self,pred = None,target = None,seq_len = None):
"""
:param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred`
:param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target`
:param seq_len: 参数映射表中 `seq_len` 的映射关系，None表示映射关系为 `seq_len` -> `seq_len`
"""
super().__init__()
self._init_param_map(pred = pred,target = target,seq_len = seq_len)
self.total = 0
self.total_pred = []
self.total_target = []
def evaluate(self,pred,target,seq_len = None):
"""
evaluate函数将针对一个批次的预测结果做评价指标的累计
:param torch.Tensor pred: 预测的tensor, tensor的形状可以是torch.Size([B,]), torch.Size([B, n_classes]),
torch.Size([B, max_len]), 或者torch.Size([B, max_len, n_classes])
:param torch.Tensor target: 真实值的tensor, tensor的形状可以是Element's can be: torch.Size([B,]),
torch.Size([B,]), torch.Size([B, max_len]), 或者torch.Size([B, max_len])
:param torch.Tensor seq_len: 序列长度标记, 标记的形状可以是None, None, torch.Size([B]), 或者torch.Size([B]).
如果mask也被传进来的话seq_len会被忽略.
"""
# pred与target的形式多种多样
if not isinstance(pred, torch.Tensor):
raise TypeError(f"`pred` in {_get_func_signature(self.evaluate)} must be torch.Tensor,"
f"got {type(pred)}.")
if not isinstance(target, torch.Tensor):
raise TypeError(f"`target` in {_get_func_signature(self.evaluate)} must be torch.Tensor,"
f"got {type(target)}.")
if seq_len is not None and not isinstance(seq_len,torch.Tensor):
raise TypeError(f"'seq_lens' in {_get_func_signature(self.evaluate)} must be torch.Tensor,"
f"got {type(seq_len)}."
)
if seq_len is not None and target.dim() > 1:
max_len = target.size(1)
masks = seq_len_to_mask(seq_len = seq_len,max_len = max_len)
else:
masks = None
if pred.dim() == target.dim():
pass
elif pred.dim() == target.dim() + 1:
pred = pred.argmax(dim = -1)
if seq_len is None and target.dim() > 1:
warnings.warn("You are not passing `seq_len` to exclude pad when calculate accuracy.")
else:
raise RuntimeError(f"In {_get_func_signature(self.evaluate)}, when pred have "
f"size:{pred.size()}, target should have size: {pred.size()} or "
f"{pred.size()[:-1]}, got {target.size()}.")
target = target.to(pred)
if masks is not None:
self.total += torch.sum(masks).item()
else:
self.total += np.prod(list(pred.size()))
self.total_pred.extend(pred.cpu().tolist())
self.total_target.extend(target.cpu().tolist())
def get_metric(self,reset = True):
p = precision_score(self.total_target,self.total_pred,average='micro')
r = recall_score(self.total_target,self.total_pred,average='micro')
f = f1_score(self.total_target,self.total_pred,average='micro')
print(classification_report(self.total_target,self.total_pred))
return {'P':p,'R':r,'F':f}
2.核心代码
from fastNLP import DataSet, Instance, Vocabulary
from fastNLP.io import ChnSentiCorpPipe, DataBundle
from tqdm import tqdm
from fastNLP.embeddings import StaticEmbedding, StackEmbedding,BertEmbedding,LSTMCharEmbedding
from fastNLP import Trainer, CrossEntropyLoss,BCELoss
import torch
from sklearn.metrics import f1_score, precision_score, recall_score, classification_report
from fastNLP.models import BertForSequenceClassification,CNNText
def get_data(file):
lines = [line for line in open(file, 'r', encoding='utf-8')]
dataset = DataSet()
for line in tqdm(lines):
assert len(line.split('\t')) == 3
# 读取内容
text = line.split('\t')[1].strip()
label = line.split('\t')[2].strip()
# 构造fastNLP框架
instance = Instance(words=list(text), target=label)
dataset.append(instance)
return dataset
def merge_data(train_set, test_set):
target_vocab = Vocabulary(padding=None, unknown=None)
target_vocab.from_dataset(train_set, test_set, field_name='target')
target_vocab.index_dataset(train_set, test_set, field_name='target')
char_vocab = Vocabulary()
char_vocab.from_dataset(train_set, test_set, field_name='words')
char_vocab.index_dataset(train_set, test_set, field_name='words')
return train_set, test_set, target_vocab, char_vocab
def evaluate(test_set, model):
probs = []
labels = []
for i in tqdm(range(len(test_set))):
word_ids = test_set.words[i]
word_ids = torch.LongTensor(word_ids)
pred = model.predict(word_ids.view(1, -1))
prob = pred['pred'].numpy()[0]
target = test_set.target[i]
probs.append(prob)
labels.append(target)
p = precision_score(labels, probs, average='micro')
r = recall_score(labels, probs, average='micro')
f1 = f1_score(labels, probs, average='micro')
print(classification_report(labels, probs))
print('P:', p)
print('R:', r)
print('F1:', f1)
train_set = get_data('./classify_data3/train.txt')
test_set = get_data('./classify_data3/test.txt')
train_set, test_set, target_vocab, char_vocab = merge_data(train_set, test_set)
fastnlp_embed = StaticEmbedding(char_vocab, model_dir_or_name='cn-char-fastnlp-100d',min_freq=2)
model_CNN = CNNText(fastnlp_embed, num_classes=27,dropout=0.1)
model_CNN.load_state_dict(torch.load('model_CNN.pth')['net'])
train_set.set_target('target')
train_set.set_input('words')
test_set.set_target('target')
test_set.set_input('words')
from fastNLP import Tester
tester = Tester(data=test_set,model = model_CNN,metrics=ClassfierMetric(),device='cpu')
tester.test()
输出:
607 out of 6280 words have frequency less than 2.
Found 5579 out of 5673 words in the pre-training embedding.
precision recall f1-score support
0 0.53 0.78 0.63 4440
1 0.50 0.48 0.49 3511
2 0.62 0.57 0.59 3016
3 0.66 0.57 0.61 2722
4 0.56 0.56 0.56 2683
5 0.65 0.63 0.64 1303
6 0.54 0.62 0.58 1268
7 0.64 0.76 0.70 1199
8 0.87 0.60 0.71 1086
9 0.88 0.67 0.76 1018
10 0.58 0.43 0.49 896
11 0.58 0.43 0.50 850
12 0.70 0.61 0.65 496
13 0.52 0.57 0.54 460
14 0.48 0.08 0.14 365
15 0.79 0.75 0.77 361
16 0.55 0.55 0.55 352
17 0.61 0.53 0.57 322
18 0.69 0.44 0.54 295
19 0.81 0.75 0.78 262
20 0.64 0.35 0.45 133
21 0.85 0.29 0.43 76
22 0.85 0.62 0.72 37
23 0.00 0.00 0.00 32
24 0.20 0.08 0.11 25
accuracy 0.59 27208
macro avg 0.61 0.51 0.54 27208
weighted avg 0.60 0.59 0.59 27208
Evaluate data in 5.79 seconds!
[tester]
ClassfierMetric: P=0.5936489267862394, R=0.5936489267862394, F=0.5936489267862394