Keras Tokenizer中的注意点

使用步骤：

1.实例化Tokenizer对象，给出最大词汇量nb_words

2.用tokenizer令牌化所有文章，把文章包装成 list(list())的形式，词或者字用空格分割

3.tokenizer.word_index会输出所有词汇与index--》也就是词表【切记如果词汇中包含大写字母，会被转成小写，后面做初始化embedding的时候，切记要转成大写】

4.embedding matrix初始化的时候，要按照词表的index来排序，采用循环来做，这个时候 要用dictionary的get方法来取values，因为用【】数组的形式，如果词汇不在词表中，会报错。当然如果try catch也是可以的。

def get_train_test_data_embeddingweights():
    input_1_list,input_2_list,label_list = get_input_and_label_list()
    qlist=[]
    qcontentlist=[]
    tokenizer = Tokenizer(nb_words=voc_size)
    token_dict = {}
#    question_content_matrix={}
    with open('question_id.csv','r',encoding='utf-8') as f:
        content_list = f.readlines()
    for i in content_list:
        values = i.split(',')
    #        print(values)
        qid = values[0]
        qlist.append(qid)
        qcontentlist.append(values[1])
    tokenizer.fit_on_texts(qcontentlist)
    sequences = tokenizer.texts_to_sequences(qcontentlist)
#    token_dict_for_emb = tokenizer.word_index.items()
    print(tokenizer.word_index)
#    print(tokenizer.word_index)
    embedding_matrix = all_embedding_dict
    embed_train_matrix = np.zeros((voc_size+1,300))
#    print(embedding_matrix)
#    print(embedding_matrix['W107878'])
    for w,i in tokenizer.word_index.items():
#        print(str(w))
#        print(embedding_matrix[str(w)])
        embedding_vector=embedding_matrix.get(w.upper())
        if embedding_vector is not None:
            embed_train_matrix[i] = embedding_vector
#        print(embedding_matrix.get(w))
        
    
    data = pad_sequences(sequences,maxlen=max_sequence_len)
    for j in range(len(content_list)):
        token_dict[content_list[j].split(',')[0]]=data[j]
    
    x1_train_list = []
    x2_train_list = []
    y_list = []
    for i1 in range(len(input_1_list)):
        x1_train_list.append(token_dict[input_1_list[i1]])
        x2_train_list.append(token_dict[input_2_list[i1]])
        y_list.append(label_list[i1])
    x1_train_list = np.array(x1_train_list)
    x2_train_list = np.array(x2_train_list)
    y_list = np.array(y_list)
#    #人工打乱
#    indices = np.arange(len(x1_train_list))
#    np.random.shuffle(indices)
#    print(indices)
#    x1_train_list = x1_train_list[indices]
#    x2_train_list = x2_train_list[indices]
#    y_list = y_list[indices]
    
#    val_split=0.8
#    
#    x_1_train = x1_train_list[:int(val_split*x1_train_list.shape[0])]
#    x_1_test = x1_train_list[int(val_split*x1_train_list.shape[0]):]
#    x_2_train = x2_train_list[:int(val_split*x2_train_list.shape[0])]
#    x_2_test = x2_train_list[int(val_split*x2_train_list.shape[0]):]
    
#    y_train = y_list[:int(val_split*y_list.shape[0])]
#    y_test = y_list[int(val_split*y_list.shape[0]):]
    return x1_train_list,x2_train_list,y_list,embed_train_matrix

这里贴一些官方文档：

、