kaggle competition 实践学习 文本分类 keras实现 模型基于yoon kim 的 Convolutional Neural Networks for Sentence Class

比赛链接实例

《 Convolutional Neural Networks for Sentence Class》论文链接

yoon kim 的《Convolutional Neural Networks for Sentence Classification》。（2014 Emnlp会议）

上面是最经典的卷积神经网络模型，于是我就用keras实现了上面的模型，

还有一下其他利用卷积神经网络的例子讲解

那个比赛就是对文本进行分类，一共五类

预处理是利用了nltk，gensim和pandas进行数据处理

其中pandas读取csv文本文件,这样就可以通过下标访问内容

nltk是对自然语言处理的一个很有用的库，pip 安装之后需要执行nltk.download()然后安装其数据库

然后这里用到了stopwords用来分词，还加入了标点符号分词，然后利用SnowballStemmer提取词的主干

代码用到了一个keras的序列处理的方法，会自动进行切割文本，当大于一定长度

然后用到了新的层Embedding，就是用来将一个词转化为n维向量，处理后一个长l的句子会变成 l*n 的矩阵，每一行代表一个单词

然后用一维的序列卷积，注意其实是长度为n的二维卷积，图像的2d卷积其实是加上维度的3维卷积，这点需要注意

下面是代码，也不难，注释掉的部分是用lstm实现的效果也差不多，在那个比赛均能达到0.62的准确度

import numpy as np
import pandas as pd

from gensim import corpora
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize 
from nltk.stem import SnowballStemmer

import keras
from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.models import Model
#from keras.layers import Dense, Activation, Convolution2D, MaxPooling2D, Flatten
from keras.layers import *
from keras.optimizers import Adam
from keras import callbacks
from keras import backend as K
from keras import metrics
from keras import regularizers

np.random.seed(0)

if __name__ == "__main__":

    #load data
    train_df = pd.read_csv('./data/train.tsv', sep='\t', header=0)
    test_df = pd.read_csv('./data/test.tsv', sep='\t', header=0)

    raw_docs_train = train_df['Phrase'].values
    raw_docs_test = test_df['Phrase'].values
    sentiment_train = train_df['Sentiment'].values
    num_labels = len(np.unique(sentiment_train))

    #text pre-processing
    stop_words = set(stopwords.words('english'))
    stop_words.update(['.', ',', '"', "'", ':', ';', '(', ')', '[', ']', '{', '}'])
    stemmer = SnowballStemmer('english')
    print stemmer

    print "pre-processing train docs..."
    processed_docs_train = []
    for doc in raw_docs_train:
       tokens = word_tokenize(doc)
       filtered = [word for word in tokens if word not in stop_words]
       stemmed = [stemmer.stem(word) for word in filtered]
       processed_docs_train.append(stemmed)
   
    print "pre-processing test docs..."
    processed_docs_test = []
    for doc in raw_docs_test:
       tokens = word_tokenize(doc)
       filtered = [word for word in tokens if word not in stop_words]
       stemmed = [stemmer.stem(word) for word in filtered]
       processed_docs_test.append(stemmed)
    print len(processed_docs_train),len(processed_docs_test)
    processed_docs_all = np.concatenate((processed_docs_train, processed_docs_test), axis=0)

    print len(processed_docs_all)
    dictionary = corpora.Dictionary(processed_docs_all)
    dictionary_size = len(dictionary.keys())
    print "dictionary size: ", dictionary_size 
    #dictionary.save('dictionary.dict')
    #corpus = [dictionary.doc2bow(doc) for doc in processed_docs]

    print "converting to token ids..."
    word_id_train, word_id_len = [], []
    for doc in processed_docs_train:
        word_ids = [dictionary.token2id[word] for word in doc]
        word_id_train.append(word_ids)
        word_id_len.append(len(word_ids))

    word_id_test, word_ids = [], []
    for doc in processed_docs_test:
        word_ids = [dictionary.token2id[word] for word in doc]
        word_id_test.append(word_ids)
        word_id_len.append(len(word_ids))
 
    seq_len = np.round((np.mean(word_id_len) + 2*np.std(word_id_len))).astype(int)
    print seq_len,np.mean(word_id_len),2*np.std(word_id_len)
    #pad sequences
    word_id_train = sequence.pad_sequences(np.array(word_id_train), maxlen=seq_len)
    word_id_test = sequence.pad_sequences(np.array(word_id_test), maxlen=seq_len)
    y_train_enc = np_utils.to_categorical(sentiment_train, num_labels)

    # #LSTM
    # print "fitting LSTM ..."
    # # model = Sequential()
    # # model.add(Embedding(dictionary_size, 128, dropout=0.2))
    # # model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))
    # # model.add(Dense(num_labels))
    # # model.add(Activation('softmax'))
    # seq_len=12
    # dictionary_size=10000
    # num_labels=10
    myInput=Input(shape=(seq_len,))
    print myInput.shape

    WORD_VECSIZE=128
    x = Embedding(output_dim=WORD_VECSIZE, input_dim=dictionary_size,dropout=0.2)(myInput)

    print x.shape
    filterNum=64
    b=Conv1D(filterNum/2, 2)(x)
    c=Conv1D(filterNum/4, 3)(x)
    d=Conv1D(filterNum/4, 4)(x)
    e=Conv1D(filterNum/4, 5)(x)
    f=Conv1D(filterNum/8, 6)(x)
    # b=Conv1D(filterNum/8, 2)(x)
    # c=Conv1D(filterNum/4, 3)(x)
    # d=Conv1D(filterNum/4, 4)(x)
    # e=Conv1D(filterNum/2, 5)(x)
    # f=Conv1D(filterNum, 6)(x)


    ba=Activation('relu')(b)
    ca=Activation('relu')(c)
    da=Activation('relu')(d)
    ea=Activation('relu')(e)
    fa=Activation('relu')(f)
    print ba.shape,fa.shape
    b2=MaxPooling1D(pool_size=(seq_len -1 ))(ba)
    c2=MaxPooling1D(pool_size=(seq_len -2 ))(ca)
    d2=MaxPooling1D(pool_size=(seq_len -3 ))(da)
    e2=MaxPooling1D(pool_size=(seq_len -4 ))(ea)
    f2=MaxPooling1D(pool_size=(seq_len -5 ))(fa)

    fb=Flatten()(b2)
    fc=Flatten()(c2)
    fd=Flatten()(d2)
    fe=Flatten()(e2)
    ff=Flatten()(f2)

    all_flatten=concatenate([fb,fc,fd,fe,ff])
    # flatten=Flatten()(all_pool)
    dp=Dropout(0.5)(all_flatten)
    # fc1=Dense(64,activation='relu')(dp)
    # dp2=Dropout(0.5)(fc1)
    out=Dense(num_labels,activation='softmax',kernel_regularizer=regularizers.l2(0.005))(dp)
    # out=Dense(NUM_CLASS,activation='softmax')(dp)

    model = Model(inputs=myInput,outputs=out)
    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  # metrics=['accuracy',metrics.categorical_accuracy])
                  metrics=['accuracy'])

    model.fit(word_id_train, y_train_enc, nb_epoch=5, batch_size=256, verbose=1)

    test_pred = model.predict(word_id_test)

    test_pred=test_pred.tolist()
    test_label =[i.index(max(i)) for i in test_pred]

    #make a submission
    test_df['Sentiment'] = np.array(test_label).reshape(-1,1) 
    header = ['PhraseId', 'Sentiment']
    test_df.to_csv('./lstm_sentiment.csv', columns=header, index=False, header=True)

后面我又参考《A C-LSTM Neural Network for Text Classification》（arXiv preprint arXiv)这篇文章改了一下，在cnn后面加上了lstm，发现效果和原来差不多。上涨了0.001.。

模型差不多，就加了一层lstm

    myInput=Input(shape=(seq_len,))
    print myInput.shape

    WORD_VECSIZE=128
    x = Embedding(output_dim=WORD_VECSIZE, input_dim=dictionary_size)(myInput)

    print x.shape
    filterNum=128
    b=Conv1D(filterNum/2, 2)(x)
    c=Conv1D(filterNum/2, 3)(x)
    d=Conv1D(filterNum, 4)(x)
    e=Conv1D(filterNum, 5)(x)
    f=Conv1D(filterNum, 6)(x)
    # b=Conv1D(filterNum/8, 2)(x)
    # c=Conv1D(filterNum/4, 3)(x)
    # d=Conv1D(filterNum/4, 4)(x)
    # e=Conv1D(filterNum/2, 5)(x)
    # f=Conv1D(filterNum, 6)(x)


    ba=Activation('relu')(b)
    ca=Activation('relu')(c)
    da=Activation('relu')(d)
    ea=Activation('relu')(e)
    fa=Activation('relu')(f)
    
    b2=MaxPooling1D(pool_size=(seq_len -1 ))(ba)
    c2=MaxPooling1D(pool_size=(seq_len -2 ))(ca)
    d2=MaxPooling1D(pool_size=(seq_len -3 ))(da)
    e2=MaxPooling1D(pool_size=(seq_len -4 ))(ea)
    f2=MaxPooling1D(pool_size=(seq_len -5 ))(fa)
    print b2.shape,f2.shape

    all_pool=concatenate([b2,c2,d2,e2,f2])
    # flatten=Flatten()(all_pool)
    # print all_pool.shape
    # res=Reshape(1)
    # print type(res),type(all_flatten)
    lstm=LSTM(128,return_sequences=True)(all_pool)

    print lstm.shape
    flatten=Flatten()(lstm)
    dp=Dropout(0.5)(flatten)
    # fc1=Dense(64,activation='relu')(dp)
    # dp2=Dropout(0.5)(fc1)
    out=Dense(num_labels,activation='softmax',kernel_regularizer=regularizers.l2(0.005))(dp)
    # out=Dense(NUM_CLASS,activation='softmax')(dp)

    model = Model(inputs=myInput,outputs=out)
    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  # metrics=['accuracy',metrics.categorical_accuracy])
                  metrics=['accuracy'])