Gensim模块训练词向量

1.准备数据与预处理

首先需要一份比较大的中文语料数据，可以考虑中文的维基百科（也可以试试搜狗的新闻语料库）。中文维基百科的打包文件地址为
https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2
中文维基百科的数据不是太大，xml的压缩文件大约1G左右。首先用 process_wiki_data.py处理这个XML压缩文件，执行：python process_wiki_data.py zhwiki-latest-pages-articles.xml.bz2 wiki.zh.text，需要安装gensim库。

import logging
import os.path
import sys

from gensim.corpora import WikiCorpus

if __name__ == "__main__":
    # os.path.basename()：返回path最后的文件名
    program = os.path.basename(sys.argv[0])

    logger = logging.getLogger(program)
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    # check and process input arguments
    if len(sys.argv) != 3:
        print(globals()['__doc__'] % locals())
        sys.exit(1)
    inp,outp = sys.argv[1:3]
    space = " "
    i = 0

    output = open(outp,"w",encoding="utf-8")
    wiki = WikiCorpus(inp,lemmatize=False,dictionary={})
    for text in wiki.get_texts():
        output.write(space.join(text)+"\n")
        i = i+1
        if(i%10000 == 0):
            logger.info("Saved "+str(i)+" articles")
    output.close()
    logger.info("Finished Saved" + str(i)+" articles")

2.繁体转简体

使用opencc-1.0.1-win64繁体转简体，生成名称为“wiki.zh.jian.text”文件。

3.结巴分词

# -*- coding:utf-8 -*-
import jieba
import jieba.analyse
import codecs

if __name__ == "__main__":
    # input文件(gbk, utf-8...)   ----decode----->   unicode  -------encode------> output文件(gbk, utf-8...)
    # 代替这繁琐的操作就是codecs.open
    f = codecs.open("wiki.zh.jian.text",'r',encoding='utf-8')
    target = codecs.open("wiki.zh.jian.seg.txt",'w',encoding='utf-8')

    line_num = 1
    line = f.readline()
    while line:
        print('---- processing', line_num, 'article----------------')
        line_seg = " ".join(jieba.cut(line))
        target.writelines(line_seg)
        line_num = line_num + 1
        line = f.readline()
    f.close()
    target.close()

4.模型训练

python train_word2vec_model.py wiki.zh.text.seg wiki.zh.text.model（保存的模型名称） wiki.zh.text.vector

import logging
import os.path
import sys
import multiprocessing

from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

if __name__ == "__main__":
    # os.path.basename()：返回path最后的文件名
    program = os.path.basename(sys.argv[0])

    logger = logging.getLogger(program)
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    # check and process input arguments
    if len(sys.argv) != 4:
        print(globals()['__doc__'] % locals())
        sys.exit(1)
    inp,outp1,outp2 = sys.argv[1:4]
    # LineSentence是按行读取文件中的每一行
    model = Word2Vec(LineSentence(inp),window=5,min_count=5,workers=multiprocessing.cpu_count())

    model.save(outp1)
    model.save_word2vec_format(outp2,binary=False)

5.模型测试

import gensim

if __name__ == '__main__':
    model = gensim.models.Word2Vec.load("wiki.zh.text.model")

    result = model.most_similar("足球")
    for e in result:
        print(e[0],e[1])