预处理tencent词向量

#数据预处理，去除数字，只保留中文
def load_tencent_word_embedding(input_file, output_file):
    n = 0
    with open('tencent.txt', 'a', encoding='utf-8', errors='ignore') as w_f:
        with open('Tencent_AILab_ChineseEmbedding.txt', 'r', encoding='utf-8', errors='ignore')as f:
            for i in tqdm(range(8824330)):# 原始
                data = f.readline()
                a = data.split()#list
                if len(a) == 201:
                    if not a[0].isdigit():
                        n = n + 1
                        w_f.write(data)
    print('total lines:',n)  # 8748463

#获取单个字词向量
def process_tencent_embedding_unigram(input_file, output_file):
    n = 0
    with open(output_file, 'a', encoding='utf-8') as g:
        with open(input_file, 'r', encoding='utf-8') as f:
            for i in tqdm(range(8748463)):
                data = f.readline()
                a = data.split()
                if i == 0:
                    g.write(data)
                else:
                    if len(a[0]) == 1:
                        n += 1
                        g.write(data)
    print('total unigram:', n)

#获取一个词(两个字)的词向量
def process_tencent_embedding_bigrams(input_file, output_file):
    n = 0
    with open(output_file, 'a', encoding='utf-8') as g:
        with open(input_file, 'r', encoding='utf-8') as f:
            for i in tqdm(range(8748463)):
                data = f.readline()
                a = data.split()
                if i == 0:
                    g.write(data)
                else:
                    if len(a[0]) == 2:
                        n += 1
                        g.write(data)
    print('total bigrams:',n)

if __name__ == '__main__':
    input_file = 'Tencent_AILab_ChineseEmbedding'
    output_file = 'tencent.txt'
    load_tencent_word_embedding(input_file, output_file)