NER数据格式转化及数据集划分
记录一次做NER时的数据格式转换过程：
1、将BMEWO–>BMESO–>BIOES
2、将数据集按8：1：1划分为train、dev、test
数据集划分方法借鉴：https://blog.csdn.net/allan2222/article/details/99672868
import random
def split(all_list, shuffle=False, ratio=0.8):
#按比例随机抽取数据，格式为每行一句话
num = len(all_list)
offset = int(num * ratio)
if num == 0 or offset < 1:
return [], all_list
if shuffle:
random.shuffle(all_list)
train = all_list[:offset]
dev_test = all_list[offset:]
return train, dev_test
def write_split(film, train, dev_test):
#将数据分为train、dev_test
infilm = open(film, 'r', encoding='utf-8')
trainfilm = open(train, 'w', encoding='utf-8')
dev_testfilm = open(dev_test, 'w', encoding='utf-8')
list = []
for datas in infilm.readlines():
datas = datas.replace('\n','')
list.append(datas)
traindatas, dev_testdatas = split(list, shuffle=True, ratio=0.8)
for traindata in traindatas:
trainfilm.write(traindata + '\n')
for dev_testdata in dev_testdatas:
dev_testfilm.write(dev_testdata + '\n')
infilm.close()
trainfilm.close()
dev_testfilm.close()
def Extract_tag(input_txt, output_txt):
'''
原格式为：vocal--label，如'菜 O'。
只要句子中含有非‘O’标签，就提取该句子。
'''
with open(input_txt,'r' ,encoding='utf-8') as f:
with open(output_txt,'a',encoding='utf-8') as g:
list = []
# list_ =[]
flag = 0
lines = f.readlines()
for line in lines:
if line.isspace() == False:#一句话未结束
list.append(line.strip())#先将该句子加入list
# for i,word in enumerate(line):
# if word.isspace()==True:#空格
else:#一句话结束，判断list该句子中是否全为‘O’
# print(list)
for words in list:
for i,word in enumerate(words):
# print(word)
if word.isspace()==True:
if words[i+1]!='O':
flag=1
break#将该list写入新的文件
else:
continue
if flag == 1:#存在不为‘O’的标签
break
if flag == 1:
for words in list:
g.write(words+'\n')
g.write('\n')
flag = 0
list = []
g.close()
f.close()
def Num_tag(input_txt):
'''
统计标签数量，原格式为：vocal--label，如'菜 O'
'''
with open(input_txt, 'r', encoding='utf-8') as f:
# sum = 46364
list = []
lines = f.readlines()
for line in lines:
for i,words in enumerate(line):
if words.isspace()==True or words == '\t':
word = line[i+1:].strip()
if word not in list:
list.append(word)
# print(list)
# print(len(list))
f.close()
return list,len(list)
def GetWords_Tags(input_txt, output_words_txt, output_tags_txt):
'''
分别获取词和标签，原格式为：vocal--label，如'菜 O'
'''
with open(input_txt, 'r', encoding='utf-8') as f:
lines = f.readlines()#获取所有行
sum = 0
words = []
tags = []
for line in lines:
if line.isspace() == False:#针对一行
for i,word in enumerate(line):
if word.isspace()==True or word==' ':
words.append(line[:i].strip())
tags.append(line[i:].strip())
sum += 1
break
else:#一句话结束
with open(output_words_txt, 'a', encoding='utf-8') as g:
for word in words:
g.write(word+' ')
g.write('\n')
words = []
with open(output_tags_txt, 'a', encoding='utf-8') as z:
for tag in tags:
z.write(tag+' ')
z.write('\n')
tags = []
with open(output_words_txt, 'a', encoding='utf-8') as g:
for word in words:
g.write(word + ' ')
g.write('\n')
with open(output_tags_txt, 'a', encoding='utf-8') as z:
for tag in tags:
z.write(tag + ' ')
z.write('\n')
# print(words)
# print(tags)
print(sum)
f.close()
g.close()
z.close()
def delete_space(input_txt, output_txt):
'''
删除多余空行，每行为一句话
'''
f = open(input_txt, 'r', encoding='utf-8')
g = open(output_txt, 'w', encoding='utf-8')
lines = f.readlines()
for line in lines:
if line.isspace()==False and line!='\n':
g.write(line)
f.close()
g.close()
def NewFileOfWordsAndTags(input_words_txt, input_tags_txt, output_txt):
'''
将词和标签合并到一句话中，输入输出每行都对应一句话。
'''
f = open(input_words_txt, 'r', encoding='utf-8')
g = open(input_tags_txt, 'r', encoding='utf-8')
z = open(output_txt, 'w', encoding='utf-8')
f_lines = f.readlines()
g_lines = g.readlines()
str = []
for i, line in enumerate(f_lines):
if line.isspace() or line == '\n':
continue
else:
# str.append(line.strip()+g_lines[i].strip())
line = line.strip()
for word in line:
str.append(word)
str.append(' ')
line_ = g_lines[i].strip()
for word_ in line_:
str.append(word_)
for i in str:
z.write(i)
z.write('\n')
str = []
f.close()
g.close()
z.close()
def SplitSentence(input_txt, output_txt):
'''
将一句话中的词和标签分隔开，输入为每行一句话。
输出格式为：vocal--label，如'菜 O'，每行一个词，一句话对应多行。
这种方法有漏洞，需要注意，比如在文章中出现'O'，会把它当作标签
'''
f = open(input_txt, 'r', encoding='utf-8')
g = open(output_txt, 'w', encoding='utf-8')
new_words = []
new_tags = []
new_line = []
for line in f.readlines(): # 针对一句话，包含实体和标签
words = line.strip().split(' ') # 针对一句话，这里是空格，也可能是\t
# print(words)
for i, word in enumerate(words): # 针对每个词
if word == 'O' or word == 'B-ORGANIZATION' or word == 'B-PERSON' or word == 'B-TIME' or word == 'B-LOCATION':
new_words = words[:i]
new_tags = words[i:]
for j, w in enumerate(new_words):
new_line.append(w + ' ' + new_tags[j])
for label in new_line:
g.write(label+'\n')
g.write('\n')
new_line = []
break
# print(new_line)
f.close()
g.close()
def SplitSentence_new(input_txt, output_txt):
f = open(input_txt, 'r', encoding='utf-8')
g = open(output_txt, 'a', encoding='utf-8')
lines = f.readlines()
for line in lines:
new_words,new_tags = line.strip().split('\t')
new_words = new_words.split()
new_tags = new_tags .split()
# print(new_words )
# print(new_tags )
for i,j in enumerate(new_words ):
g.write(j + ' ' + new_tags [i] + '\n')
g.write('\n')
return 0
if __name__ == '__main__':
input_txt = './data/RenminNER/BIOES/train.txt'
output_txt = './data/RenminNER/BIOES/new_train.txt'
SplitSentence(input_txt, output_txt)