cnn、rnn相结合进行文本分类
主要参考代码思路：
https://github.com/jiegzhan/multi-class-text-classification-cnn-rnn
cnn和rnn结合一起进行文本分类主要思路如下:
data--->batch iter-->cnn input-->embedding--->卷积--->池化--->rnn输入--->lstm cell--softmax
在前面的博客已经提到如何把文本数据转化了一个batch iter的形式，下面贴上关于cnn-rnn文本分类的一些代码:
基本配置:
class TCNNRNNConfig(object): # 模型参数 embedding_dim = 64 # 词向量维度 seq_length = 300 # 序列长度 num_classes = 2 # 类别数 num_filters = 256 # 卷积核数目 kernel_size = 5 # 卷积核尺寸 vocab_size = 130000 # 词汇表达小 max_pool_size=4 #最大的pool层 hidden_dim = 128 # 全连接层神经元 dropout_keep_prob = 0.8 # dropout保留比例 learning_rate = 1e-3 # 学习率 hidden_unit=256 #lstm神经元的个数 batch_size = 128 # 每批训练大小 num_epochs = 20 # 总迭代轮次 print_per_batch = 100 # 每多少轮输出一次结果 multi_kernel_size = '3,4,5' l2_reg_lambda = 0.0
模型代码：
#!/usr/bin/python# -*- coding: utf-8 -*-import tensorflow as tfimport numpy as npclass TextCnnRnn(object): def __init__(self,config): self.config=config self.input_x=tf.placeholder(tf.int32,[None, self.config.seq_length],name="input_x") self.input_y=tf.placeholder(tf.float32,[None, self.config.num_classes],name="inpyt_y") self.keep_prob=tf.placeholder(tf.float32,None,name='keep_prob') self.pad = tf.placeholder(tf.float32, [None, 1, self.config.embedding_dim, 1], name='pad') self.l2_loss = tf.constant(0.0) self.real_len = tf.placeholder(tf.int32, [None], name='real_len') self.filter_sizes = list(map(int, self.config.multi_kernel_size.split(","))) self.cnnrnn() def input_embedding(self): """词嵌套""" with tf.device('/cpu:0'): embedding =tf.get_variable("embedding",[self.config.vocab_size,self.config.embedding_dim]) _input = tf.nn.embedding_lookup(embedding, self.input_x) _input_expanded = tf.expand_dims(_input, -1) return _input_expanded def cnnrnn(self): emb=self.input_embedding() pooled_concat = [] reduced = np.int32(np.ceil((self.config.seq_length) * 1.0 / self.config.max_pool_size)) for i, filter_size in enumerate(self.filter_sizes): with tf.name_scope('conv-maxpool-%s' % filter_size): # Zero paddings so that the convolution output have dimension batch x sequence_length x emb_size x channel num_prio = (filter_size - 1) // 2 num_post = (filter_size - 1) - num_prio pad_prio = tf.concat([self.pad] * num_prio, 1) pad_post = tf.concat([self.pad] * num_post, 1) emb_pad = tf.concat([pad_prio, emb, pad_post], 1) filter_shape = [filter_size, self.config.embedding_dim, 1, self.config.num_filters] W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name='W') b = tf.Variable(tf.constant(0.1, shape=[self.config.num_filters]), name='b') conv = tf.nn.conv2d(emb_pad, W, strides=[1, 1, 1, 1], padding='VALID', name='conv') h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu') # Maxpooling over the outputs pooled = tf.nn.max_pool(h, ksize=[1, self.config.max_pool_size, 1, 1], strides=[1, self.config.max_pool_size, 1, 1], padding='SAME', name='pool') pooled = tf.reshape(pooled, [-1, reduced, self.config.num_filters]) pooled_concat.append(pooled) pooled_concat = tf.concat(pooled_concat, 2) pooled_concat = tf.nn.dropout(pooled_concat, self.keep_prob) # lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=self.config.hidden_unit) # lstm_cell = tf.nn.rnn_cell.GRUCell(num_units=self.config.hidden_unit) lstm_cell = tf.contrib.rnn.GRUCell(num_units=self.config.hidden_unit) # lstm_cell = tf.nn.rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=self.dropout_keep_prob) lstm_cell = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=self.keep_prob) self._initial_state = lstm_cell.zero_state(self.config.batch_size, tf.float32) # inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(1, reduced, pooled_concat)] inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(pooled_concat, num_or_size_splits=int(reduced), axis=1)] # outputs, state = tf.nn.rnn(lstm_cell, inputs, initial_state=self._initial_state, sequence_length=self.real_len) #outputs, state = tf.contrib.rnn.static_rnn(lstm_cell, inputs, initial_state=self._initial_state, # sequence_length=self.real_len) outputs, state=tf.nn.static_rnn( lstm_cell, inputs,self._initial_state,sequence_length=self.real_len) # Collect the appropriate last words into variable output (dimension = batch x embedding_size) output = outputs[0] with tf.variable_scope('Output'): tf.get_variable_scope().reuse_variables() one = tf.ones([1, self.config.hidden_unit], tf.float32) for i in range(1, len(outputs)): ind = self.real_len < (i + 1) ind = tf.to_float(ind) ind = tf.expand_dims(ind, -1) mat = tf.matmul(ind, one) output = tf.add(tf.multiply(output, mat), tf.multiply(outputs[i], 1.0 - mat)) with tf.name_scope('score'): self.W = tf.Variable(tf.truncated_normal([self.config.hidden_unit, self.config.num_classes], stddev=0.1), name='W') b = tf.Variable(tf.constant(0.1, shape=[self.config.num_classes]), name='b') self.l2_loss += tf.nn.l2_loss(W) self.l2_loss += tf.nn.l2_loss(b) self.scores = tf.nn.xw_plus_b(output, self.W, b, name='scores') self.pred_y = tf.nn.softmax(self.scores, name="pred_y") tf.add_to_collection('pred_network', self.pred_y) self.predictions = tf.argmax(self.scores, 1, name='predictions') with tf.name_scope('loss'): losses = tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y, logits=self.scores) # only named arguments accepted self.loss = tf.reduce_mean(losses) + self.config.l2_reg_lambda * self.l2_loss with tf.name_scope("optimize"): # 优化器 optimizer = tf.train.AdamOptimizer( learning_rate=self.config.learning_rate) self.optim = optimizer.minimize(self.loss) with tf.name_scope('accuracy'): correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1)) self.acc = tf.reduce_mean(tf.cast(correct_predictions, "float"), name='accuracy') with tf.name_scope('num_correct'): correct = tf.equal(self.predictions, tf.argmax(self.input_y, 1)) self.num_correct = tf.reduce_sum(tf.cast(correct, 'float'))
run代码:
#!/usr/bin/python# -*- coding: utf-8 -*-from cnn_rnn_model import TextCnnRnnfrom configuration import TCNNRNNConfigfrom data_utils_cut import preocess_file,batch_iterimport timeimport tensorflow as tfimport osimport numpy as npfrom datetime import timedeltatrainpath="/Users/shuubiasahi/Desktop/tensorflow/adx/"def run_epoch(cnnrnnmodel=True): # 载入数据 print('Loading data...') start_time = time.time() x_train, y_train, words = preocess_file(data_path=trainpath+"cnn.txt") if cnnrnnmodel: print('Using CNNRNN model...') config = TCNNRNNConfig() config.vocab_size = len(words) print("vocab_size is:", config.vocab_size) model = TextCnnRnn(config) tensorboard_dir = '/Users/shuubiasahi/Desktop/tensorflow/boardlog' end_time = time.time() time_dif = end_time - start_time time_dif = timedelta(seconds=int(round(time_dif))) print('Time usage:', time_dif) print('Constructing TensorFlow Graph...') session = tf.Session() session.run(tf.global_variables_initializer()) saver = tf.train.Saver() # 配置 tensorboard tf.summary.scalar("loss", model.loss) tf.summary.scalar("accuracy", model.acc) if not os.path.exists(tensorboard_dir): os.makedirs(tensorboard_dir) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter(tensorboard_dir) writer.add_graph(session.graph) # 生成批次数据 print('Generating batch...') batch_train = batch_iter(list(zip(x_train, y_train)), config.batch_size, config.num_epochs) def feed_data(batch): """准备需要喂入模型的数据""" x_batch, y_batch = zip(*batch) feed_dict = { model.input_x: x_batch, model.input_y: y_batch, model.real_len:real_len(x_batch) } return feed_dict, len(x_batch) def real_len(batches): return [np.ceil(np.argmin(batch + [0]) * 1.0 / config.max_pool_size) for batch in batches] def evaluate(x_, y_): """ 模型评估 一次运行所有的数据会OOM，所以需要分批和汇总 """ batch_eval = batch_iter(list(zip(x_, y_)), 128, 1) total_loss = 0.0 total_acc = 0.0 cnt = 0 for batch in batch_eval: feed_dict, cur_batch_len = feed_data(batch) feed_dict[model.keep_prob] = 1.0 loss, acc = session.run([model.loss, model.acc], feed_dict=feed_dict) total_loss += loss * cur_batch_len total_acc += acc * cur_batch_len cnt += cur_batch_len return total_loss / cnt, total_acc / cnt # 训练与验证 print('Training and evaluating...') start_time = time.time() print_per_batch = config.print_per_batch for i, batch in enumerate(batch_train): feed_dict, lenbatch = feed_data(batch) feed_dict[model.keep_prob] = config.dropout_keep_prob feed_dict[model.pad]=np.zeros([lenbatch, 1, config.embedding_dim, 1]) if i % 5 == 0: # 每5次将训练结果写入tensorboard scalar s = session.run(merged_summary, feed_dict=feed_dict) writer.add_summary(s, i) if i % print_per_batch == print_per_batch - 1: # 每200次输出在训练集和验证集上的性能 loss_train, acc_train = session.run([model.loss, model.acc], feed_dict=feed_dict) #loss, acc = evaluate(x_val, y_val) 验证机暂时不需要 # 时间 end_time = time.time() time_dif = end_time - start_time time_dif = timedelta(seconds=int(round(time_dif))) msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},'\ + ' Time: {3}' print(msg.format(i + 1, loss_train, acc_train, time_dif)) # if i%10==0 and i>0: # graph=tf.graph_util.convert_variables_to_constants(session,session.graph_def,["keep_prob","input_x","score/pred_y"]) # tf.train.write_graph(graph,".","/Users/shuubiasahi/Desktop/tensorflow/modelsavegraph/graph.db",as_text=False) if i%500==0 and i>0: graph = tf.graph_util.convert_variables_to_constants(session, session.graph_def, ["keep_prob","real_len","pad", "input_x", "score/pred_y"]) if cnnrnnmodel: tf.train.write_graph(graph, ".", trainpath+"graphcnnrnn.model", as_text=False) print("模型在第{0}步已经保存".format(i)) session.run(model.optim, feed_dict=feed_dict) # 运行优化 # 最后在测试集上进行评估 session.close()if __name__ == '__main__': run_epoch()
简单的结果分析:
Using CNNRNN model...vocab_size is: 160238Time usage: 0:00:35Constructing TensorFlow Graph...2017-10-30 23:22:18.426329: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations.2017-10-30 23:22:18.426342: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations.2017-10-30 23:22:18.426346: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations.2017-10-30 23:22:18.426351: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations.Generating batch...Training and evaluating...Iter: 100, Train Loss: 0.66, Train Acc: 71.09%, Time: 0:02:47Iter: 200, Train Loss: 0.65, Train Acc: 61.72%, Time: 0:05:38
迭代几百步相比单纯的用cnn、bi-lstm实际效果是很差了，可能文本本身的特征已经够明显，再用这种反而效果会变差吧，cnn这种相当于一个超级n-gram，bi-lstm正反两面捕捉文本上下文的信息进行信息输出，之前在GitHub上看到别人做文本分类，cnn、bilstm这种量效果是最佳。。。由于电脑原因并没有迭代很多步，哪天用gpu试试吧