import re
res = [
re.compile(r'<[^>]+>',re.S),
re.compile(r'\&[a-zA-Z]+;',re.S),
re.compile(r'\n|\t',re.S),
re.compile(r'\[|\]|\(|\)',re.S),
# re.compile(r'[\.\!\/`_,$%^*(+\"\')]+|[+——()?“”!,;;<=>~~{|}。?、~@#¥……&*()]+|[/.,"=:_-\{\}-、,-、;\\[];\-():。《》〉]+',re.S),
re.compile(r'【.*】',re.S), #去除括号和括号内的文字
re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
r'localhost|' #localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
# re.compile(r'[0-9A-Za-z]+', re.S) #去除数字和英文
# re.compile(r'[0-9]+', re.S) # 仅去除数字
]
with open('cnews.test.txt','r+',encoding='utf-8') as f_raw:
with open('cnews.test1.txt','a+',encoding='utf-8') as f_aim:
for line in f_raw:
label=line.split('\t')[0]
sent=line.split('\t')[1]
f_aim.write(label)
f_aim.write('\t')
for r in res:
sent = r.sub('', sent)
sent = re.sub(r"[^\u4e00-\u9fa5]", ",", sent)
sent = re.sub('!{2,}', '!', sent)
sent = re.sub('! {2,}', '!', sent)
sent = re.sub('?{2,}', '!', sent)
sent = re.sub('。{2,}', '。', sent)
sent = re.sub(',{2,}', ',', sent)
sent = re.sub('\s{2,}', '', sent)
f_aim.write(sent)
f_aim.write('\n')