readExcel.py
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 5 10:03:17 2017
@author: Sean Chang
"""
import xlrd
emodic = {"PA":6,"PE":6,"PD":0,"PH":0,"PG":0,"PB":0,"PK":0,"NA":3,"NB":5,"NJ":5 \
,"NH":5,"PF":5,"NI":1,"NC":1,"NG":1,"NE":2,"ND":2,"NN":2,"NK":2,"NL":2,"PC":4}
def EmoLexicon(filename):
data = xlrd.open_workbook(filename)
table = data.sheets()[0]
nrows = table.nrows
emolist = []
emoindex = []
for i in range(1,nrows):
emolist.append(table.row_values(i)[0])
emoindex.append(emodic[table.row_values(i)[4].strip()])
return emolist,emoindex
if __name__ == "__main__":
emolist,emoindex = EmoLexicon('emotion ontology.xlsx')
rankNB.py
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 7 06:41:17 2017
@author: Sean Chang
"""
import xml.dom.minidom
import jieba
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
stop_words = ["的", "一", "不", "在", "人", "有", "是", "为", "以", "于", "上", "他", "而",
"后", "之", "来", "及", "了", "因", "下", "可", "到", "由", "这", "与", "也",
"此", "但", "并", "个", "其", "已", "无", "小", "我", "们", "起", "最", "再",
"今", "去", "好", "只", "又", "或", "很", "亦", "某", "把", "那", "你", "乃",
"它","要", "将", "应", "位", "新", "两", "中", "更", "我们", "自己", "没有", "“", "”",
"，", "（", "）", " ",'[',']',' ','~','。','!']
emo = ['like','fear','disgust','anger','surprise','sadness','happiness']
def readXML(filename):
# 使用minidom解析器打开 XML 文档
DOMTree = xml.dom.minidom.parse(filename)
collection = DOMTree.documentElement
weibos = collection.getElementsByTagName("weibo")
Ncount = 0 #count of all the class
sen_lst = []
for weibo in weibos:
sentence = weibo.getElementsByTagName('sentence')
for e in sentence:
lst = []
if e.getAttribute('opinionated')=='Y':
lst.append(e.childNodes[0].data)
emotion1 = e.getAttribute('emotion-1-type')
Ncount = Ncount + 1
emotion2 = e.getAttribute('emotion-2-type')
if emotion2 != 'none':
Ncount = Ncount + 1
lst.append([emotion1,emotion2])
sen_lst.append(lst)
emotionlist = [[],[],[],[],[],[],[]]
for e in sen_lst:
if 'like' in e[1]:
emotionlist[0].append(e[0])
if 'fear' in e[1]:
emotionlist[1].append(e[0])
if 'disgust' in e[1]:
emotionlist[2].append(e[0])
if 'anger' in e[1]:
emotionlist[3].append(e[0])
if 'surprise' in e[1]:
emotionlist[4].append(e[0])
if 'sadness' in e[1]:
emotionlist[5].append(e[0])
if 'happiness' in e[1]:
emotionlist[6].append(e[0])
return emotionlist,Ncount
def readTestXML(filename):
testdata = []
testlabel = []
DOMTree = xml.dom.minidom.parse(filename)
collection = DOMTree.documentElement
weibos = collection.getElementsByTagName("weibo")
for weibo in weibos:
emotion1 = weibo.getAttribute("emotion-type1")
emotion2 = weibo.getAttribute("emotion-type2")
sen = ""
if emotion1 in emo:
sentence = weibo.getElementsByTagName('sentence')
for e in sentence:
sen += e.childNodes[0].data
testdata.append(sen)
label1 = emo.index(emotion1)
if emotion2 in emo:
label2 = emo.index(emotion2)
else:
label2 = -1
testlabel.append([label1,label2])
return testdata,testlabel
def lcm(x,y): # very fast
s = x*y
while y: x, y = y, x%y
return s/x
#solve the bias distribution problem
def adjustData(emotionlist):
l = []
for e in emotionlist:
l.append(len(e))
m = max(l)
maxindex = l.index(m)
Ncount= 0
for i in range(len(emotionlist)):
if maxindex == i:
#emotionlist[i] *= 2
Ncount += l[i]
else:
emotionlist[i] *= (m//l[i])
Ncount += l[i]*(m//l[i])
return emotionlist,Ncount
def createData(emotionlist,Ncount):
data = []
label = []
for i in range(len(emotionlist)):
for e in emotionlist[i]:
data.append(e)
label.append(i)
return data,label
def segmentWord(cont):
c = []
for i in cont:
a = list(jieba.cut(i))
b = " ".join(a)
c.append(b)
return c
def showresult(rst):
c = ['like','fear','disgust','anger','surprise','sadness','happiness']
rs = sorted(rst)
max1 = rst.index(rs[-1])
max2 = rst.index(rs[-2])
return c[max1],c[max2]
def rank(lst):
sortl = sorted(lst,reverse=False)
r = []
for e in lst:
r.append(sortl.index(e))
return r
def train(data,label):
vectorizer = CountVectorizer()
content = segmentWord(data)
opinion = label
tfidf = vectorizer.fit_transform(content)
clf = MultinomialNB()
clf.fit(tfidf, opinion)
return clf,vectorizer
def test(sentence,clf,vectorizer):
docs = [" ".join(list(jieba.cut(sentence)))]
new_tfidf = vectorizer.transform(docs)
predicted = clf.predict_log_proba(new_tfidf)
return predicted[0]
if __name__ == "__main__":
'''
emotionlist,Ncount = readXML("NLPCC.xml")
emotionlist,Ncount = adjustData(emotionlist)
data,label = createData(emotionlist,Ncount)
clf,vectorizer = train(data,label)
print(rank(test("老鼠怕猫？",clf,vectorizer)))
'''
testdata,testlabel = readTestXML("NLPtest.xml")
binaryNB.py
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 2 19:23:04 2017
@author: Sean Chang
"""
from xml.dom.minidom import parse
import xml.dom.minidom
import jieba
from sklearn.naive_bayes import MultinomialNB
import rankNB
from sklearn.feature_extraction.text import CountVectorizer
stop_words = ["的", "一", "不", "在", "人", "有", "是", "为", "以", "于", "上", "他", "而",
"后", "之", "来", "及", "了", "因", "下", "可", "到", "由", "这", "与", "也",
"此", "但", "并", "个", "其", "已", "无", "小", "我", "们", "起", "最", "再",
"今", "去", "好", "只", "又", "或", "很", "亦", "某", "把", "那", "你", "乃",
"它","要", "将", "应", "位", "新", "两", "中", "更", "我们", "自己", "没有", "“", "”",
"，", "（", "）", " ",'[',']',' ','~','。','!']
emo = ['like','fear','disgust','anger','surprise','sadness','happiness']
def readXML(filename):
# 使用minidom解析器打开 XML 文档
DOMTree = xml.dom.minidom.parse(filename)
collection = DOMTree.documentElement
# 在集合中获取所有电影
weibos = collection.getElementsByTagName("weibo")
Ncount = 0 #count of all the class
sen_lst = []
for weibo in weibos:
sentence = weibo.getElementsByTagName('sentence')
for e in sentence:
lst = []
if e.getAttribute('opinionated')=='Y':
lst.append(e.childNodes[0].data)
emotion1 = e.getAttribute('emotion-1-type')
Ncount = Ncount + 1
emotion2 = e.getAttribute('emotion-2-type')
if emotion2 != 'none':
Ncount = Ncount + 1
lst.append([emotion1,emotion2])
sen_lst.append(lst)
emotionlist = [[],[],[],[],[],[],[]]
for e in sen_lst:
if 'like' in e[1]:
emotionlist[0].append(e[0])
if 'fear' in e[1]:
emotionlist[1].append(e[0])
if 'disgust' in e[1]:
emotionlist[2].append(e[0])
if 'anger' in e[1]:
emotionlist[3].append(e[0])
if 'surprise' in e[1]:
emotionlist[4].append(e[0])
if 'sadness' in e[1]:
emotionlist[5].append(e[0])
if 'happiness' in e[1]:
emotionlist[6].append(e[0])
return emotionlist,Ncount
def create7Data(emotionlist,Ncount):
rst = []
for i in range(7):
data = []
label = []
data = [e for e in emotionlist[i]]
for j in range(7):
if j!=i:
data += [e for e in emotionlist[j]]
label = [1]*len(emotionlist[i])
label += [-1]*(Ncount - len(emotionlist[i]))
rst.append([data,label])
return rst
def segmentWord(cont):
c = []
for i in cont:
a = list(jieba.cut(i))
b = " ".join(a)
c.append(b)
return c
def testSentence(rsti,docs):
vectorizer = CountVectorizer()
content = segmentWord(rsti[0])
opinion = rsti[1]
tfidf = vectorizer.fit_transform(content)
clf = MultinomialNB()
clf.fit(tfidf, opinion)
new_tfidf = vectorizer.transform(docs)
predicted = clf.predict(new_tfidf)
return predicted[0]
if __name__ == "__main__":
emotionlist,Ncount = readXML("NLPCC.xml")
'''
likedata,likelabel = createLike(emotionlist,Ncount)
emotionlist,Ncount = parseXML.adjustData(emotionlist)
rst = create7Data(emotionlist,Ncount)
like_data_feature,word_feature = prepareForTraining(rst[0][0])
print(testSentence(like_data_feature,word_feature,rst[0][1],"今天真的好开心"))
'''
emotionlist,Ncount = rankNB.adjustData(emotionlist)
rst = create7Data(emotionlist,Ncount)
docs = [" ".join(list(jieba.cut("好怀念 [泪] 我记得这照片好像是@Christine_HJ 是CC拍的")))]
print(testSentence(rst[5],docs))
main.py
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 5 06:28:52 2017
@author: Sean Chang
"""
import binaryNB
import rankNB
import jieba
import readExcel
import random
if __name__ == "__main__":
emo = ['like','fear','disgust','anger','surprise','sadness','happiness']
#read the train document
emotionlist,Ncount = rankNB.readXML("NLPCC.xml")
#emotionlist,Ncount = rankNB.adjustData(emotionlist)
#adjust the train data for average distribution
#emotionlist,Ncount = parseXML.adjustData(emotionlist)
#load emotion lexicon
print("load emotion lexicon")
emolist,emoindex = readExcel.EmoLexicon('emotion ontology.xlsx')
data,label = rankNB.createData(emotionlist,Ncount)
#train the data for first ranking
clf,vectorizer = rankNB.train(data,label)
#merge the 7 binary classification train data
rst7data = binaryNB.create7Data(emotionlist,Ncount)
'''
testdata,testlabel = rankNB.readTestXML("NLPtest.xml")
#randomly choose 20 sentence for testing
randomlen = testdata.__len__()
indexList = range(randomlen)
randomIndex = random.sample(indexList, 100)
test_data = []
test_label = []
for i in randomIndex:
test_data.append(testdata[i])
test_label.append(testlabel[i])
'''
#sen_counter :the ith sentence
sen_counter = 0
#correct number
correct = 0
test_data=['生气生气生气吃惊吃惊吃惊！']
test_label = [[3,4]]
for string in test_data:
print("calculating sentence",sen_counter+1)
print(string)
docs = [" ".join(list(jieba.cut(string)))]
#merge the first rank list by testing
r = rankNB.rank(rankNB.test(string,clf,vectorizer))
print(r)
#updating ranking
print("updating ranking")
#7 binary classification for updating
for i in range(7):
r[i] += binaryNB.testSentence(rst7data[i],docs)
print(r)
#cut the test sentence
word_list = jieba.cut(string,cut_all=True)
#update the corresponding index in rank []
for word in word_list:
if word in emolist:
i = emolist.index(word)
r[emoindex[i]] += 1
print(r)
# find the top 2 emotion:
copyr = [e for e in r]
m1 = max(copyr)
index1 = r.index(m1)
del copyr[index1]
m2 = max(copyr)
if m1 != m2:
index2 = r.index(m2)
else:
r[index1] += 1
index2 = r.index(m2)
r[index1] -= 1
l = test_label[sen_counter]
if -1 not in l:
if (index1 in l) and (index2 in l):
correct += 1
else:
if (index1 in l) or (index2 in l):
correct += 1
sen_counter += 1
print("now correct:",correct)
print("precision is:",correct/len(test_data))