lstm情感分析（keras）详解

苏神的https://spaces.ac.cn/archives/3863中wordembedding版代码详解 

# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
import jieba


def cut_word(x):
    s = list(jieba.cut(x)) 
    return s

neg = pd.read_excel('neg.xls',header=None,index=None)
pos = pd.read_excel('pos.xls',header=None,index=None)
pos['label'] = 1#给每行赋值
neg['label'] = 0
data = pd.concat([pos,neg],ignore_index = True)
print('评论总数%d'%len(data))
cw = lambda x: list(jieba.cut(x)) #定义分词函数
data['words'] = data[0].apply(cw) #pd库 对data[0]每一列分词添加
#data['words'] = data[0].apply(lambda x: list(jieba.cut(x)))

maxlen = 100
min_count = 5

content = []
for i in data['words']:#pd库 可遍历该列数据
    content.extend(i)

word_num = pd.Series(content).value_counts()#这里的中文是索引
word_num = word_num[word_num >= min_count]#去除词频小于5的数据（索引就没了）
word_num[:] = list(range(1,len(word_num)+1))#将词频改为数字顺序
word_num[''] = 0#因为数字作为索引要0 所以补齐
word_set = set(word_num.index)#对索引求集合（这里的索引是文字）

def doc2num(s, maxlen): #转换为满维度的词向量，不同于word2vec
    s = [i for i in s if i in word_set]
    s = s[:maxlen] + ['']*max(0, maxlen-len(s))
    return list(word_num[s])

data['doc2num'] = data['words'].apply(lambda s: doc2num(s, maxlen))#对每一个内容进行转换为词向量并添加

#手动打乱数据,防止数据褒贬顺序影响
idx = list(range(len(data)))
np.random.shuffle(idx)
data = data.loc[idx]

#按keras的输入要求来生成数据
x = np.array(list(data['doc2num']))#神经网络输入需要np.array
y = np.array(list(data['label']))
y = y.reshape((-1,1)) #调整标签形状,转置

from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Embedding, LSTM

#建立模型
model = Sequential()
model.add(Embedding(len(word_num), 256, input_length=maxlen))#它不是作为降维使用，而是作为一种特征表示使用
model.add(LSTM(128))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])

batch_size = 128
train_num = 15000


model.fit(x[:train_num], y[:train_num], batch_size = batch_size, nb_epoch=30,validation_split=0.1)

model.evaluate(x[train_num:], y[train_num:], batch_size = batch_size)

def predict_one(s): #单个句子的预测函数
    s = np.array(doc2num(s, maxlen))
    s = s.reshape((1, s.shape[0]))
    return model.predict_classes(s, verbose=0)[0][0]