背景
在nlp领域，预训练模型bert可谓是红得发紫。
但现在能搜到的大多数都是pytorch写的框架，而且大多都是单输出模型。
所以，本文以 有相互关系的多层标签分类 为背景，用keras设计了多输出、参数共享的模型。
keras_bert基础应用
def batch_iter(data_path, cat_to_id, tokenizer, batch_size=64, shuffle=True):
"""生成批次数据"""
keras_bert_iter = get_keras_bert_iterator(data_path, cat_to_id, tokenizer)
while True:
data_list = []
for _ in range(batch_size):
data = next(keras_bert_iter)
data_list.append(data)
if shuffle:
random.shuffle(data_list)
indices_list = []
segments_list = []
label_index_list = []
for data in data_list:
indices, segments, label_index = data
indices_list.append(indices)
segments_list.append(segments)
label_index_list.append(label_index)
yield [np.array(indices_list), np.array(segments_list)], np.array(label_index_list)
def get_model(label_list):
K.clear_session()
bert_model = load_trained_model_from_checkpoint(bert_paths.config, bert_paths.checkpoint, seq_len=text_max_length) #加载预训练模型
for l in bert_model.layers:
l.trainable = True
input_indices = Input(shape=(None,))
input_segments = Input(shape=(None,))
bert_output = bert_model([input_indices, input_segments])
bert_cls = Lambda(lambda x: x[:, 0])(bert_output) # 取出[CLS]对应的向量用来做分类
output = Dense(len(label_list), activation='softmax')(bert_cls)
model = Model([input_indices, input_segments], output)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(1e-5), #用足够小的学习率
metrics=['accuracy'])
print(model.summary())
return model
early_stopping = EarlyStopping(monitor='val_acc', patience=3) #早停法，防止过拟合
plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5, patience=2) #当评价指标不在提升时，减少学习率
checkpoint = ModelCheckpoint('trained_model/keras_bert_THUCNews.hdf5', monitor='val_acc',verbose=2, save_best_only=True, mode='max', save_weights_only=True) #保存最好的模型
def get_step(sample_count, batch_size):
step = sample_count // batch_size
if sample_count % batch_size != 0:
step += 1
return step
batch_size = 4
train_step = get_step(train_sample_count, batch_size)
dev_step = get_step(dev_sample_count, batch_size)
train_dataset_iterator = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, batch_size)
dev_dataset_iterator = batch_iter(r"data/keras_bert_dev.txt", cat_to_id, tokenizer, batch_size)
model = get_model(categories)
#模型训练
model.fit(
train_dataset_iterator,
steps_per_epoch=train_step,
epochs=10,
validation_data=dev_dataset_iterator,
validation_steps=dev_step,
callbacks=[early_stopping, plateau, checkpoint],
verbose=1
)多输出、参数共享的模型设计
def batch_iter(data_path, cat_to_id, tokenizer, second_label_list, batch_size=64, shuffle=True):
"""生成批次数据"""
keras_bert_iter = get_keras_bert_iterator(data_path, cat_to_id, tokenizer, second_label_list)
while True:
data_list = []
for _ in range(batch_size):
data = next(keras_bert_iter)
data_list.append(data)
if shuffle:
random.shuffle(data_list)
indices_list = []
segments_list = []
label_index_list = []
second_label_list = []
for data in data_list:
indices, segments, label_index, second_label = data
indices_list.append(indices)
segments_list.append(segments)
label_index_list.append(label_index)
second_label_list.append(second_label)
yield [np.array(indices_list), np.array(segments_list)], [np.array(label_index_list), np.array(second_label_list)]
def get_model(label_list, second_label_list):
K.clear_session()
bert_model = load_trained_model_from_checkpoint(bert_paths.config, bert_paths.checkpoint, seq_len=text_max_length) #加载预训练模型
for l in bert_model.layers:
l.trainable = True
input_indices = Input(shape=(None,))
input_segments = Input(shape=(None,))
bert_output = bert_model([input_indices, input_segments])
bert_cls = Lambda(lambda x: x[:, 0])(bert_output) # 取出[CLS]对应的向量用来做分类
output = Dense(len(label_list), activation='softmax')(bert_cls)
output_second = Dense(len(second_label_list), activation='softmax')(bert_cls)
model = Model([input_indices, input_segments], [output, output_second])
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(1e-5), #用足够小的学习率
metrics=['accuracy'])
print(model.summary())
return model
batch_size = 4
train_step = get_step(train_sample_count, batch_size)
dev_step = get_step(dev_sample_count, batch_size)
train_dataset_iterator = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, second_label_list, batch_size)
dev_dataset_iterator = batch_iter(r"data/keras_bert_dev.txt", cat_to_id, tokenizer, second_label_list, batch_size)
model = get_model(categories, second_label_list)
#模型训练
model.fit(
train_dataset_iterator,
steps_per_epoch=train_step,
epochs=10,
validation_data=dev_dataset_iterator,
validation_steps=dev_step,
callbacks=[early_stopping, plateau, checkpoint],
verbose=1
)附录
全部源码
import os
import sys
import re
from collections import Counter
import random
from tqdm import tqdm
import numpy as np
import tensorflow.keras as keras
from keras_bert import load_vocabulary, load_trained_model_from_checkpoint, Tokenizer, get_checkpoint_paths
from keras_bert.layers import MaskedGlobalMaxPool1D
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
from keras.metrics import top_k_categorical_accuracy
from keras.layers import *
from keras.callbacks import *
from keras.models import Model
import keras.backend as K
from keras.optimizers import Adam
from keras.utils import to_categoricaldata_path = "000_text_classifier_tensorflow_textcnn/THUCNews/"
text_max_length = 512
bert_paths = get_checkpoint_paths(r"chinese_L-12_H-768_A-12")构建原数据文本迭代器
def _read_file(filename):
"""读取一个文件并转换为一行"""
with open(filename, 'r', encoding='utf-8') as f:
s = f.read().strip().replace('\n', '。').replace('\t', '').replace('\u3000', '')
return re.sub(r'。+', '。', s)def get_data_iterator(data_path):
for category in os.listdir(data_path):
category_path = os.path.join(data_path, category)
for file_name in os.listdir(category_path):
yield _read_file(os.path.join(category_path, file_name)), categoryit = get_data_iterator(data_path)next(it)('竞彩解析：日本美国争冠死磕 两巴相逢必有生死。周日受注赛事，女足世界杯决赛、美洲杯两场1/4决赛毫无疑问是全世界球迷和彩民关注的焦点。本届女足世界杯的最大黑马日本队能否一黑到底，创造亚洲奇迹？女子足坛霸主美国队能否再次“灭黑”成功，成就三冠伟业？巴西、巴拉圭冤家路窄，谁又能笑到最后？诸多谜底，在周一凌晨就会揭晓。日本美国争冠死磕。本届女足世界杯，是颠覆与反颠覆之争。夺冠大热门东道主德国队1/4决赛被日本队加时赛一球而“黑”，另一个夺冠大热门瑞典队则在半决赛被日本队3:1彻底打垮。而美国队则捍卫着女足豪强的尊严，在1/4决赛，她们与巴西女足苦战至点球大战，最终以5:3淘汰这支迅速崛起的黑马球队，而在半决赛，她们更是3:1大胜欧洲黑马法国队。美日两队此次世界杯进程惊人相似，小组赛前两轮全胜，最后一轮输球，1/4决赛同样与对手90分钟内战成平局，半决赛竟同样3:1大胜对手。此次决战，无论是日本还是美国队夺冠，均将创造女足世界杯新的历史。两巴相逢必有生死。本届美洲杯，让人大跌眼镜的事情太多。巴西、巴拉圭冤家路窄似乎更具传奇色彩。两队小组赛同分在B组，原本两个出线大热门，却双双在前两轮小组赛战平，两队直接交锋就是2:2平局，结果双双面临出局危险。最后一轮，巴西队在下半场终于发威，4:2大胜厄瓜多尔后来居上以小组第一出线，而巴拉圭最后一战还是3:3战平委内瑞拉获得小组第三，侥幸凭借净胜球优势挤掉A组第三名的哥斯达黎加，获得一个八强席位。在小组赛，巴西队是在最后时刻才逼平了巴拉圭，他们的好运气会在淘汰赛再显神威吗？巴拉圭此前3轮小组赛似乎都缺乏运气，此番又会否被幸运之神补偿一下呢？。另一场美洲杯1/4决赛，智利队在C组小组赛2胜1平以小组头名晋级八强；而委内瑞拉在B组是最不被看好的球队，但竟然在与巴西、巴拉圭同组的情况下，前两轮就奠定了小组出线权，他们小组3战1胜2平保持不败战绩，而入球数跟智利一样都是4球，只是失球数比智利多了1个。但既然他们面对强大的巴西都能保持球门不失，此番再创佳绩也不足为怪。',
'彩票')
构建标签表
def read_category(data_path):
"""读取分类目录，固定"""
categories = os.listdir(data_path)
cat_to_id = dict(zip(categories, range(len(categories))))
return categories, cat_to_idcategories, cat_to_id = read_category(data_path)cat_to_id{'彩票': 0,
'家居': 1,
'游戏': 2,
'股票': 3,
'科技': 4,
'社会': 5,
'财经': 6,
'时尚': 7,
'星座': 8,
'体育': 9,
'房产': 10,
'娱乐': 11,
'时政': 12,
'教育': 13}
categories['彩票',
'家居',
'游戏',
'股票',
'科技',
'社会',
'财经',
'时尚',
'星座',
'体育',
'房产',
'娱乐',
'时政',
'教育']
构建训练、验证、测试集
def build_dataset(data_path, train_path, dev_path, test_path):
data_iter = get_data_iterator(data_path)
with open(train_path, 'w', encoding='utf-8') as train_file, \
open(dev_path, 'w', encoding='utf-8') as dev_file, \
open(test_path, 'w', encoding='utf-8') as test_file:
for text, label in tqdm(data_iter):
radio = random.random()
if radio < 0.8:
train_file.write(text + "\t" + label + "\n")
elif radio < 0.9:
dev_file.write(text + "\t" + label + "\n")
else:
test_file.write(text + "\t" + label + "\n")# build_dataset(data_path, r"data/keras_bert_train.txt", r"data/keras_bert_dev.txt", r"data/keras_bert_test.txt")获取数据集样本个数
def get_sample_num(data_path):
count = 0
with open(data_path, 'r', encoding='utf-8') as data_file:
for line in tqdm(data_file):
count += 1
return counttrain_sample_count = get_sample_num(r"data/keras_bert_train.txt")668858it [00:09, 67648.27it/s]
dev_sample_count = get_sample_num(r"data/keras_bert_dev.txt")83721it [00:01, 61733.96it/s]
test_sample_count = get_sample_num(r"data/keras_bert_test.txt")83496it [00:01, 72322.53it/s]
train_sample_count, dev_sample_count, test_sample_count(668858, 83721, 83496)
构建数据迭代器
def get_text_iterator(data_path):
with open(data_path, 'r', encoding='utf-8') as data_file:
for line in data_file:
data_split = line.strip().split('\t')
if len(data_split) != 2:
print(line)
continue
yield data_split[0], data_split[1]it = get_text_iterator(r"data/keras_bert_train.txt")next(it)('竞彩解析：日本美国争冠死磕 两巴相逢必有生死。周日受注赛事，女足世界杯决赛、美洲杯两场1/4决赛毫无疑问是全世界球迷和彩民关注的焦点。本届女足世界杯的最大黑马日本队能否一黑到底，创造亚洲奇迹？女子足坛霸主美国队能否再次“灭黑”成功，成就三冠伟业？巴西、巴拉圭冤家路窄，谁又能笑到最后？诸多谜底，在周一凌晨就会揭晓。日本美国争冠死磕。本届女足世界杯，是颠覆与反颠覆之争。夺冠大热门东道主德国队1/4决赛被日本队加时赛一球而“黑”，另一个夺冠大热门瑞典队则在半决赛被日本队3:1彻底打垮。而美国队则捍卫着女足豪强的尊严，在1/4决赛，她们与巴西女足苦战至点球大战，最终以5:3淘汰这支迅速崛起的黑马球队，而在半决赛，她们更是3:1大胜欧洲黑马法国队。美日两队此次世界杯进程惊人相似，小组赛前两轮全胜，最后一轮输球，1/4决赛同样与对手90分钟内战成平局，半决赛竟同样3:1大胜对手。此次决战，无论是日本还是美国队夺冠，均将创造女足世界杯新的历史。两巴相逢必有生死。本届美洲杯，让人大跌眼镜的事情太多。巴西、巴拉圭冤家路窄似乎更具传奇色彩。两队小组赛同分在B组，原本两个出线大热门，却双双在前两轮小组赛战平，两队直接交锋就是2:2平局，结果双双面临出局危险。最后一轮，巴西队在下半场终于发威，4:2大胜厄瓜多尔后来居上以小组第一出线，而巴拉圭最后一战还是3:3战平委内瑞拉获得小组第三，侥幸凭借净胜球优势挤掉A组第三名的哥斯达黎加，获得一个八强席位。在小组赛，巴西队是在最后时刻才逼平了巴拉圭，他们的好运气会在淘汰赛再显神威吗？巴拉圭此前3轮小组赛似乎都缺乏运气，此番又会否被幸运之神补偿一下呢？。另一场美洲杯1/4决赛，智利队在C组小组赛2胜1平以小组头名晋级八强；而委内瑞拉在B组是最不被看好的球队，但竟然在与巴西、巴拉圭同组的情况下，前两轮就奠定了小组出线权，他们小组3战1胜2平保持不败战绩，而入球数跟智利一样都是4球，只是失球数比智利多了1个。但既然他们面对强大的巴西都能保持球门不失，此番再创佳绩也不足为怪。',
'彩票')
token_dict = load_vocabulary(bert_paths.vocab)tokenizer = Tokenizer(token_dict)def get_keras_bert_iterator(data_path, cat_to_id, tokenizer):
while True:
data_iter = get_text_iterator(data_path)
for text, category in data_iter:
indices, segments = tokenizer.encode(first=text, max_len=text_max_length)
yield indices, segments, cat_to_id[category]it = get_keras_bert_iterator(r"data/keras_bert_train.txt", cat_to_id, tokenizer)# next(it)构建批次数据迭代器
def batch_iter(data_path, cat_to_id, tokenizer, batch_size=64, shuffle=True):
"""生成批次数据"""
keras_bert_iter = get_keras_bert_iterator(data_path, cat_to_id, tokenizer)
while True:
data_list = []
for _ in range(batch_size):
data = next(keras_bert_iter)
data_list.append(data)
if shuffle:
random.shuffle(data_list)
indices_list = []
segments_list = []
label_index_list = []
for data in data_list:
indices, segments, label_index = data
indices_list.append(indices)
segments_list.append(segments)
label_index_list.append(label_index)
yield [np.array(indices_list), np.array(segments_list)], np.array(label_index_list)it = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, batch_size=1)# next(it)it = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, batch_size=2)next(it)([array([[ 101, 4993, 2506, ..., 131, 123, 102],
[ 101, 2506, 3696, ..., 1139, 125, 102]]),
array([[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0]])],
array([0, 0]))
定义base模型
def get_model(label_list):
K.clear_session()
bert_model = load_trained_model_from_checkpoint(bert_paths.config, bert_paths.checkpoint, seq_len=text_max_length) #加载预训练模型
for l in bert_model.layers:
l.trainable = True
input_indices = Input(shape=(None,))
input_segments = Input(shape=(None,))
bert_output = bert_model([input_indices, input_segments])
bert_cls = Lambda(lambda x: x[:, 0])(bert_output) # 取出[CLS]对应的向量用来做分类
output = Dense(len(label_list), activation='softmax')(bert_cls)
model = Model([input_indices, input_segments], output)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(1e-5), #用足够小的学习率
metrics=['accuracy'])
print(model.summary())
return modelearly_stopping = EarlyStopping(monitor='val_acc', patience=3) #早停法，防止过拟合
plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5, patience=2) #当评价指标不在提升时，减少学习率
checkpoint = ModelCheckpoint('trained_model/keras_bert_THUCNews.hdf5', monitor='val_acc',verbose=2, save_best_only=True, mode='max', save_weights_only=True) #保存最好的模型模型训练
def get_step(sample_count, batch_size):
step = sample_count // batch_size
if sample_count % batch_size != 0:
step += 1
return stepbatch_size = 4
train_step = get_step(train_sample_count, batch_size)
dev_step = get_step(dev_sample_count, batch_size)
train_dataset_iterator = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, batch_size)
dev_dataset_iterator = batch_iter(r"data/keras_bert_dev.txt", cat_to_id, tokenizer, batch_size)
model = get_model(categories)
#模型训练
model.fit(
train_dataset_iterator,
steps_per_epoch=train_step,
epochs=10,
validation_data=dev_dataset_iterator,
validation_steps=dev_step,
callbacks=[early_stopping, plateau, checkpoint],
verbose=1
)Model: "functional_5"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 512)] 0
__________________________________________________________________________________________________
input_2 (InputLayer) [(None, 512)] 0
__________________________________________________________________________________________________
functional_3 (Functional) (None, 512, 768) 101677056 input_1[0][0]
input_2[0][0]
__________________________________________________________________________________________________
lambda (Lambda) (None, 768) 0 functional_3[0][0]
__________________________________________________________________________________________________
dense (Dense) (None, 14) 10766 lambda[0][0]
==================================================================================================
Total params: 101,687,822
Trainable params: 101,687,822
Non-trainable params: 0
__________________________________________________________________________________________________
None
Epoch 1/10
5/167215 [..............................] - ETA: 775:02:36 - loss: 0.4064 - accuracy: 0.9000
---------------------------------------------------------------------------
多输出模型
构建数据迭代器
second_label_list = [0, 1, 2]def get_keras_bert_iterator(data_path, cat_to_id, tokenizer, second_label_list):
while True:
data_iter = get_text_iterator(data_path)
for text, category in data_iter:
indices, segments = tokenizer.encode(first=text, max_len=text_max_length)
yield indices, segments, cat_to_id[category], random.choice(second_label_list)it = get_keras_bert_iterator(r"data/keras_bert_train.txt", cat_to_id, tokenizer, second_label_list)# next(it)def batch_iter(data_path, cat_to_id, tokenizer, second_label_list, batch_size=64, shuffle=True):
"""生成批次数据"""
keras_bert_iter = get_keras_bert_iterator(data_path, cat_to_id, tokenizer, second_label_list)
while True:
data_list = []
for _ in range(batch_size):
data = next(keras_bert_iter)
data_list.append(data)
if shuffle:
random.shuffle(data_list)
indices_list = []
segments_list = []
label_index_list = []
second_label_list = []
for data in data_list:
indices, segments, label_index, second_label = data
indices_list.append(indices)
segments_list.append(segments)
label_index_list.append(label_index)
second_label_list.append(second_label)
yield [np.array(indices_list), np.array(segments_list)], [np.array(label_index_list), np.array(second_label_list)]it = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, second_label_list, batch_size=2)next(it)([array([[ 101, 4993, 2506, ..., 131, 123, 102],
[ 101, 2506, 3696, ..., 1139, 125, 102]]),
array([[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0]])],
[array([0, 0]), array([0, 0])])
定义模型
def get_model(label_list, second_label_list):
K.clear_session()
bert_model = load_trained_model_from_checkpoint(bert_paths.config, bert_paths.checkpoint, seq_len=text_max_length) #加载预训练模型
for l in bert_model.layers:
l.trainable = True
input_indices = Input(shape=(None,))
input_segments = Input(shape=(None,))
bert_output = bert_model([input_indices, input_segments])
bert_cls = Lambda(lambda x: x[:, 0])(bert_output) # 取出[CLS]对应的向量用来做分类
output = Dense(len(label_list), activation='softmax')(bert_cls)
output_second = Dense(len(second_label_list), activation='softmax')(bert_cls)
model = Model([input_indices, input_segments], [output, output_second])
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(1e-5), #用足够小的学习率
metrics=['accuracy'])
print(model.summary())
return modelearly_stopping = EarlyStopping(monitor='val_acc', patience=3) #早停法，防止过拟合
plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5, patience=2) #当评价指标不在提升时，减少学习率
checkpoint = ModelCheckpoint('trained_model/muilt_keras_bert_THUCNews.hdf5', monitor='val_acc',verbose=2, save_best_only=True, mode='max', save_weights_only=True) #保存最好的模型模型训练
batch_size = 4
train_step = get_step(train_sample_count, batch_size)
dev_step = get_step(dev_sample_count, batch_size)
train_dataset_iterator = batch_iter(r"data/keras_bert_train.txt", cat_to_id, tokenizer, second_label_list, batch_size)
dev_dataset_iterator = batch_iter(r"data/keras_bert_dev.txt", cat_to_id, tokenizer, second_label_list, batch_size)
model = get_model(categories, second_label_list)
#模型训练
model.fit(
train_dataset_iterator,
steps_per_epoch=train_step,
epochs=10,
validation_data=dev_dataset_iterator,
validation_steps=dev_step,
callbacks=[early_stopping, plateau, checkpoint],
verbose=1
)Model: "functional_5"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 512)] 0
__________________________________________________________________________________________________
input_2 (InputLayer) [(None, 512)] 0
__________________________________________________________________________________________________
functional_3 (Functional) (None, 512, 768) 101677056 input_1[0][0]
input_2[0][0]
__________________________________________________________________________________________________
lambda (Lambda) (None, 768) 0 functional_3[0][0]
__________________________________________________________________________________________________
dense (Dense) (None, 14) 10766 lambda[0][0]
__________________________________________________________________________________________________
dense_1 (Dense) (None, 3) 2307 lambda[0][0]
==================================================================================================
Total params: 101,690,129
Trainable params: 101,690,129
Non-trainable params: 0
__________________________________________________________________________________________________
None
Epoch 1/10
7/167215 [..............................] - ETA: 1829:52:33 - loss: 3.1260 - dense_loss: 1.4949 - dense_1_loss: 1.6311 - dense_accuracy: 0.6429 - dense_1_accuracy: 0.3571
