python正则表达式处理文本内容

import re


res = [
        re.compile(r'<[^>]+>',re.S),
        re.compile(r'\&[a-zA-Z]+;',re.S),
        re.compile(r'\n|\t',re.S),
        re.compile(r'\[|\]|\(|\)',re.S),
        # re.compile(r'[\.\!\/`_,$%^*(+\"\')]+|[+——()?“”!,;;<=>~~{|}。?、~@#¥……&*()]+|[/.,"=:_-\{\}-、,-、;\\[];\-():。《》〉]+',re.S),
        re.compile(r'【.*】',re.S),    #去除括号和括号内的文字
        re.compile(
        r'^(?:http|ftp)s?://' # http:// or https://
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
        r'localhost|' #localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
        r'(?::\d+)?' # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
        # re.compile(r'[0-9A-Za-z]+', re.S)   #去除数字和英文
        # re.compile(r'[0-9]+', re.S)   # 仅去除数字
        ]
with open('cnews.test.txt','r+',encoding='utf-8') as f_raw:
    with open('cnews.test1.txt','a+',encoding='utf-8') as f_aim:
        for line in f_raw:
            label=line.split('\t')[0]
            sent=line.split('\t')[1]
            f_aim.write(label)
            f_aim.write('\t')
            for r in res:
                sent = r.sub('', sent)
            sent = re.sub(r"[^\u4e00-\u9fa5]", ",", sent)
            sent = re.sub('!{2,}', '!', sent)
            sent = re.sub('! {2,}', '!', sent)
            sent = re.sub('?{2,}', '!', sent)
            sent = re.sub('。{2,}', '。', sent)
            sent = re.sub(',{2,}', ',', sent)
            sent = re.sub('\s{2,}', '', sent)
            f_aim.write(sent)
            f_aim.write('\n')

 

你可能感兴趣的:(数据处理)