模型训练的两种方法
方法1:自定义训练过程：
import configparser
import pprint
from model.SingleRNNModel import SingleRNNModel
import tensorflow as tf
from model.TextCNN import TextCNN
from utils.path_utils import get_full_path
from utils.read_batch_data import get_data_by_batch_size, get_data
class SingleRNNModelTest:
def __init__(self,
epoch=3,
batch_size=100,
embedding_size=256,
learning_rate=0.001,
model_path="model_version",
sentence_vocb_length=20,
fill_vocab='TTTTTT',
vocab_file_path="data/vocab_clean.txt"
):
self.optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
self.loss = tf.keras.losses.BinaryCrossentropy(reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE)
self.epoch = epoch
self.batch_size = batch_size
self.model_path = model_path
self.sentence_vocb_length = sentence_vocb_length
self.fill_vocab = fill_vocab
# self.model = SingleRNNModel(vocab=get_full_path(vocab_file_path), embedding_size=embedding_size,
# sentence_length=sentence_vocb_length)
self.model = TextCNN(vocab=get_full_path(vocab_file_path), embedding_size=embedding_size,hidden_size=20,
sentence_length=sentence_vocb_length)
self.summary_writer = tf.summary.create_file_writer('./tensorboard/news_label_model/{}'.format(model_path))
# 更具有参考意义
# self.auc = tf.keras.metrics.AUC(curve='PR', name='p-r')
# 准确率：a = (TP+TN)/(TP+FP+TN+FN)
self.accuracy = tf.keras.metrics.BinaryAccuracy(threshold=0.5)
# # 精准率：TP/(TP+FP)
# self.precision = tf.keras.metrics.PrecisionAtRecall(0.v2_5)
# # 查全率（召回率）：R = TP/(TP+FN)
# self.recall = tf.keras.metrics.RecallAtPrecision(0.v2_5)
# 返回结果的顺序性指标
def train(self):
# ========================== Create dataset =======================
original_accuracy = 0
version = 0
count = 0
batch_step = 0
for each in range(self.epoch):
print("epoch:{}".format(each))
if original_accuracy>0.999:
break
for train_x, train_y in get_data_by_batch_size("data/train_data/train_data.txt", self.batch_size,
self.sentence_vocb_length, self.fill_vocab):
count += 1
print("model train path is {}, count is {}".format(self.model_path, count))
with tf.GradientTape() as tape:
y_predict = self.model(train_x)
outer_loss = self.loss(train_y, y_predict)
batch_step += 1
# 记录损失值
with self.summary_writer.as_default(): # 指定记录器
tf.summary.scalar("train_loss", outer_loss, step=batch_step)
# 参数调整
grads = tape.gradient(outer_loss, self.model.variables)
self.optimizer.apply_gradients(grads_and_vars=zip(grads, self.model.variables))
# 记录测试集损失值
if batch_step % 14 == 0:
# 准确率统计（TP+TN）/(TP+FP+TN+FN)
self.accuracy.reset_state()
for x, y in get_data_by_batch_size("data/train_data/test_data.txt", self.batch_size,
self.sentence_vocb_length, self.fill_vocab):
test_x = x
test_y = y
y_predict_test = self.model(test_x)
self.accuracy.update_state(test_y, y_predict_test)
with self.summary_writer.as_default(): # 指定记录器
current_accuracy = self.accuracy.result().numpy()
tf.summary.scalar("test_accuracy", self.accuracy.result().numpy(), step=batch_step)
if current_accuracy > original_accuracy:
pprint.pprint("save model, model version is {},current accuracy is {}".format(version,current_accuracy))
original_accuracy = current_accuracy
version += 1
tf.keras.models.save_model(self.model,
get_full_path("data/{0}/{1}".format(self.model_path, version)),
signatures={"call": self.model.call
}
)
if original_accuracy > 0.999:
break
# 准确率记录 AUC
if __name__ == '__main__':
# =============================== GPU ==============================
gpu = tf.config.experimental.list_physical_devices(device_type='GPU')
print("gpu message:{}".format(gpu))
# If you have GPU, and the value is GPU serial number.
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
epoch = 20
batch_size = 1000
config = configparser.ConfigParser()
config.read(get_full_path('conf/config.txt'), encoding="utf-8")
sentence_vocb_length = config.getint("lstm_model", "sentence_vocab_lenth")
embedding_size = 216
learning_rate = 0.001
train_instance = SingleRNNModelTest(epoch=epoch, batch_size=batch_size, sentence_vocb_length=sentence_vocb_length,
embedding_size=embedding_size, learning_rate=learning_rate)
train_instance.train()
方法2：复用已经封装好的fit方法：
import configparser
import pprint
from model.SingleRNNModel import SingleRNNModel
import tensorflow as tf
from model.TextCNN import TextCNN
from utils.path_utils import get_full_path
from utils.read_batch_data import get_data_by_batch_size, get_data
class SingleRNNModelTest:
def __init__(self,
epoch=3,
batch_size=100,
embedding_size=256,
learning_rate=0.001,
model_path="model_version",
sentence_vocb_length=20,
fill_vocab='TTTTTT',
vocab_file_path="data/vocab_clean.txt"
):
self.optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
self.loss = tf.keras.losses.BinaryCrossentropy(reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE)
self.epoch = epoch
self.batch_size = batch_size
self.model_path = model_path
self.sentence_vocb_length = sentence_vocb_length
self.fill_vocab = fill_vocab
self.model = TextCNN(vocab=get_full_path(vocab_file_path), embedding_size=embedding_size, hidden_size=20,
sentence_length=sentence_vocb_length)
self.summary_writer = tf.summary.create_file_writer('./tensorboard/news_label_model/{}'.format(model_path))
def train(self):
# ========================== Create dataset =======================
train_x,train_y = get_data("data/train_data/train_data.txt", self.sentence_vocb_length, self.fill_vocab)
board = tf.keras.callbacks.TensorBoard(log_dir=get_full_path("data/fit_log/graph"), write_graph=True)
model_save = tf.keras.callbacks.ModelCheckpoint(get_full_path("data/fit_log/fit_model"), monitor="val_loss",
mode="min")
self.model.compile(optimizer=self.optimizer, loss=self.loss,
metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0.5),
tf.keras.metrics.AUC(curve='PR', name='p-r'),
tf.keras.metrics.AUC(curve='ROC', name='ROC'),
])
self.model.fit(x=train_x,y=train_y,batch_size=1000, epochs=15,callbacks=[board, model_save])
if __name__ == '__main__':
# =============================== GPU ==============================
gpu = tf.config.experimental.list_physical_devices(device_type='GPU')
print("gpu message:{}".format(gpu))
# If you have GPU, and the value is GPU serial number.
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
epoch = 20
batch_size = 1000
config = configparser.ConfigParser()
config.read(get_full_path('conf/config.txt'), encoding="utf-8")
sentence_vocb_length = config.getint("lstm_model", "sentence_vocab_lenth")
embedding_size = 216
learning_rate = 0.001
train_instance = SingleRNNModelTest(epoch=epoch, batch_size=batch_size, sentence_vocb_length=sentence_vocb_length,
embedding_size=embedding_size, learning_rate=learning_rate)
train_instance.train()