统计学习第11章 习题
习题11.1
习题11.2
理解前后向算法，而且我前面的隐马尔可夫里的前后向算法结果是相同的
习题11.3
理解如何从特征*权重对应x,y的概率，可以与前面的判别模型对比
# -*- coding:utf8 -*-
"""
说明：CRF 梯度下降（不算完整的）
作者：nkenen
时间：2020年5月10日
"""
import numpy as np
class CRF():
def __init__(self,path):
self.traindata = []
self.Y_set = set()
self.X_set = set()
self.px = {}#x的经验分布
self.pxy = {}#p(x,y)联合概率
self.totalcount = 0.0
self.loaddata(path)
self.f = self.createfi()
self.wi = np.array([[0.0]*len(self.f)])
assert self.wi.shape[1] == len(self.f)
def loaddata(self,path):
with open(path,encoding='utf8') as f:
for line in f:
if line == [] or line == '\n':
return
line = line.strip()
linelist = line.split()
line_l = []
line_l.append([None,'start'])
for l in linelist:
xy = l.split('/')
line_l.append(xy)
if xy[0] not in self.px:
self.px[xy[0]] = 0.0
self.px[xy[0]]+=1
if xy[0] not in self.pxy and xy[1] not in self.pxy:
self.pxy[xy[0],xy[1]] = 0.0
self.pxy[xy[0],xy[1]]+=1
self.totalcount += 1
self.X_set.add(xy[0])
self.Y_set.add(xy[1])
line_l.append([None,'end'])
self.traindata.append(line_l)
#最后预测序列时需要定位yi，转为list
self.X_set = list(self.X_set)
self.Y_set = list(self.Y_set)
"""
print(self.Y_set)
print(self.X_set)
print(self.px)
print(self.pxy)
print(self.totalcount)
"""
pass
#创建特征
def createfi(self):
feature1 = [lambda y_pre,y,x,i,_yp='start',_yi=yi:1 if y_pre==_yp and y==_yi and i==1 else 0 for yi in self.Y_set]
feature2 = [lambda y_pre,y,x,i,_yp=yp,_yi=yi:1 if y_pre==_yp and y==_yi and i>1 else 0 for yp in self.Y_set for yi in self.Y_set]
feature3 = [lambda y_pre,y,x,i,_yp=yp,_yi='end':1 if y_pre==_yp and y==_yi and i==(len(x)-1) else 0 for yp in self.Y_set]
feature4 = [lambda y_pre,y,x,i,_yi=yi,_xi=xi:1 if y==_yi and x[i]==_xi else 0 for xi in self.X_set for yi in self.Y_set]
return feature1+feature2+feature3+feature4
#计算fi，我一直是从0开始，range可以限制，我忘记了
def computfi(self,x_v,y_v,fi):
fi_sum = 0.0
for i,yi in enumerate(y_v):
if i == 0:
continue
fi_sum += fi(y_v[i-1],yi,x_v,i)
return fi_sum
#计算g(w)的公式，感觉公式我有可能理解错误，有错误请联系我
#计算ep（fi），书中p231，11.51
def computep(self,x_v,y_v,fi):
def computpxy(x_v,y_v):
pxy = 1.0
for i,yi in enumerate(y_v):
if i == 0:
continue
if yi == 'end':
pxy *= 1.0
else:
pxy *=self.pxy[x_v[i],yi]/self.totalcount
return pxy
return computpxy(x_v, y_v)*self.computfi(x_v, y_v, fi)
#前向算法，本次没有使用
def forward(self,x_v,M_M):
A_M = []
for i in range(len(x_v)):
if i ==0 :
ai = np.array([1.0]*len(self.Y_set))
else:
ai = A_M[i-1].dot(M_M[i-1])
A_M.append(ai)
return A_M
#后向算法，本次没有使用
def backward(self,x_v,M_M):
B_M = [None]*len(x_v)
x_len = len(x_v)
for i in range(x_len-1,-1,-1):
if i ==len(x_v)-1 :
bi = np.array([1.0]*len(self.Y_set))
else:
bi = M_M[i-1].dot(B_M[i+1].T)
B_M[i] = bi
return B_M
#矩阵Mi中的W
def comptW(self,y_pre,y,x_v,i):
f_v = []
for fi in self.f:
f_v.append(fi(y_pre,y,x_v,i))
return np.exp(self.wi.dot(np.array([f_v]).T)[0][0])
#计算ewp时，我使用了11.49公式的上半部分，和11.24的下半部分
#因为是特定的y组合序列，若有错误请指正
def computewp(self,x_v,y_v,fi):
def computM(x_v):
M_M = []
for i,_ in enumerate(x_v):
if i ==0 :
continue
if i==1:
mi = np.array([[self.comptW('start', yi, x_v, i) if j==0 else 0.0 for k,yi in enumerate(self.Y_set)]for j,yp in enumerate(self.Y_set)])
elif i == len(x_v)-1:
mi = np.array([[1 if k==0 else 0 for k,yi in enumerate(self.Y_set)]for j,yp in enumerate(self.Y_set)])
else:
mi = np.array([[self.comptW(yp, yi, x_v, i) for k,yi in enumerate(self.Y_set)]for j,yp in enumerate(self.Y_set)])
M_M.append(mi)
return M_M
"""
crf不完整体现：我并没有将所有可能路径算出，我假定了start=None
因为我是在做习题11.4时才理解，M可以进行选择，由start和end的值去选择，其实用书中例题一样也可以，
在计算pw(y|x)时没有加上前后向矩阵是错误，由于电脑计算慢，这边就不加上了，由于没有验证请自己重新编写
"""
def computpwxy(x_v,y_v):
#先获得所有的Mi矩阵
M_M = computM(x_v)
#用矩阵求Z比较容易
Z = None
for i,M in enumerate(M_M):
if i == 0:
Z= M
Z = Z.dot(M)
#这时候Z[0][0]为所有的路径概率和
pwxy = 1.0
for i,yi in enumerate(y_v):
if i ==0:
continue
pwxy *= self.comptW(y_v[i-1], yi, x_v, i)
return pwxy/Z[0][0]
#这是X序列的xi出现概率积，这边我不清楚理解的对不对
def computpxv(x_v):
pxv = 1.0
for i,xi in enumerate(x_v):
if i == 0 or i ==len(x_v)-1:
continue
pxv *= self.px[xi]/self.totalcount
return pxv
return computpxv(x_v)*computpwxy(x_v, y_v)*self.computfi(x_v, y_v, fi)
#判断不多的梯度下降，因为我电脑实在太垃圾了，计算太久了还是简化终止判断
def traing(self,max_iter=100):
for _ in range(max_iter):
gw = []
for fi in self.f:
gwi = 0.0
for xy_v in self.traindata:
x_v = [xy[0] for xy in xy_v]
y_v = [xy[1] for xy in xy_v]
gwi += self.computewp(x_v, y_v, fi) - self.computep(x_v, y_v, fi)
gw.append(gwi)
gw = np.array([gw])
fanshu = np.linalg.norm(gw,ord=2)
print(fanshu)
#在这只判断gw的范数是否收敛，我没有收敛成功过
if fanshu<=1e-10:
break
self.wi -= gw
print(self.wi)
#维特比预测
def predict(self,x_test):
def computFi(y_pre,yi,x_v,i):
Fi = []
for fi in self.f:
Fi.append(fi(y_pre ,yi,x_v,i))
return np.array([Fi]).T
delta = []
psi = []
for i in range(len(x_test)):
deltai = []
psii = []
if i == 0:
continue
#由start开始
elif i==1:
for yj in self.Y_set:
deltai.append(self.wi.dot(computFi('start', yj, x_test, i))[0][0])
#由end结束
elif i==len(x_test)-1:
max_prob = 0.0
max_yj = None
for j,yj in enumerate(self.Y_set):
prob = delta[i-2][j]+self.wi.dot(computFi(yj, 'end', x_test, i))[0][0]
max_yj = yj if prob>max_prob else max_yj
max_prob = prob if prob>max_prob else max_prob
deltai.append(max_prob)
psii.append(max_yj)
else:
for yl in self.Y_set:
max_prob = 0.0
max_yj = None
for j,yj in enumerate(self.Y_set):
prob = delta[i-2][j]+self.wi.dot(computFi(yj, yl, x_test, i))[0][0]
max_yj = yj if prob>max_prob else max_yj
max_prob = prob if prob>max_prob else max_prob
deltai.append(max_prob)
psii.append(max_yj)
psi.append(psii)
delta.append(deltai)
print(psi)
path = []
j = 0
#回溯
for i in range(len(x_test)-1,-1,-1):
if i <=1:
break
y_end = psi[i-1][j]
j = self.Y_set.index(y_end)
path.insert(0,y_end)
print(path)
def main():
crf = CRF("data.txt")
crf.traing(3)
#开始值结束值为None
crf.predict([None,'我', '十分', '高兴', '地', '致以', '诚挚', '的', '问候', '！',None])
if __name__ == "__main__":
main()习题11.4
