机器学习技法作业二，Experiment with Bagging Ridge Regression.
13~14题为linear LSSVM算法，代码如下，与RBF kernel的LSSVM不同的只是kernel的计算部分：
import numpy as np
from math import *
import matplotlib.pyplot as plt
def loaddata(file):
f = open(file)
try:
lines = f.readlines()
finally:
f.close()
example_num = len(lines)
dimension = len(lines[0].strip().split())
features = np.zeros((example_num, dimension))
labels = np.zeros((example_num, 1))
features[:,0] = 1
for index, line in enumerate(lines):
item = lines[index].strip().split()
features[index,1:] = [float(feature) for feature in item[0:-1]]
labels[index] = float(item[-1])
return features, labels
def LSSVM(X, Y, gama, lamb):
N = len(Y)
Lamb = np.eye(N) * lamb ##lambda矩阵
K = np.zeros((N,N)) ##初始化K矩阵 N×N
for i in range(N): ##计算K矩阵
for j in range(N):
#K[i,j] = exp(-gama * np.dot((X[i] - X[j]).T, (X[i] - X[j]))) ##RBF
K[i,j] = np.dot(X[i].T, X[j]) ##linear
beta = np.dot(np.linalg.inv(Lamb + K), Y)
return beta
def predict(Xmodel, Xtest, Ytest, beta, gama):
p = np.zeros((len(Ytest),1))
py = np.zeros((len(Ytest),1))
temp = np.zeros((len(Xmodel),1))
for i in range(len(Ytest)): #i--第i个测试数据
for j in range(len(Xmodel)): #j--第j个训练数据
temp[j] = np.dot(Xtest[i].T, Xmodel[j]) #linear
#temp[j] = exp(-gama * np.dot((Xtest[i] - Xmodel[j]).T, (Xtest[i] - Xmodel[j]))) #temp--1～N训练数据与第i个数据求Kernel
p[i] = np.dot(temp.T, beta)
py[i] = [1 if np.dot(temp.T, beta)>=0 else -1] #sigma(1~N):beta(N)*Kenel(xn,x)
accuracy = sum(py == Ytest)/len(Ytest)
return p,accuracy
X, Y = loaddata('hw2_lssvm_all.dat.txt')
gama = 0.125
lamb = 0.01
beta1 = LSSVM(X[:400,:], Y[:400], gama, lamb)
lamb = 0.1
beta2 = LSSVM(X[:400,:], Y[:400], gama, lamb)
lamb = 1
beta3 = LSSVM(X[:400,:], Y[:400], gama, lamb)
lamb = 10
beta4 = LSSVM(X[:400,:], Y[:400], gama, lamb)
lamb = 100
beta5 = LSSVM(X[:400,:], Y[:400], gama, lamb)
p,acc = predict(X[:400,:], X[401:,:], Y[401:], beta3, gama)
##acc = array([0.63636364])
15~16题是LSSVM与Bootstrapping结合，代码如下：
import numpy as np
from math import *
import matplotlib.pyplot as plt
def loaddata(file):
f = open(file)
try:
lines = f.readlines()
finally:
f.close()
example_num = len(lines)
dimension = len(lines[0].strip().split()) #features添加了x0 = 1
features = np.zeros((example_num, dimension))
labels = np.zeros((example_num, 1))
features[:,0] = 1 #初始化features的x0 = 1
for index, line in enumerate(lines):
item = lines[index].strip().split()
features[index,1:] = [float(feature) for feature in item[0:-1]]
labels[index] = float(item[-1])
return features, labels
class Bagging(object):
def __init__(self, interation, X, Y): ##X, Y为训练数据，为了引入训练数据维度
self.__inter = interation
self.__beta = np.zeros((interation, len(Y), 1)) ##保存gt的参数beta
self.__Xbar = np.zeros((interation, len(Y), X.shape[1])) ##保存bootstrap采样得到的训练数据
self.__Ybar = np.zeros((interation, len(Y), 1))
def Bootstrap(self, X, Y): ##进行bootstrap采样，共进行self.__inter次
N = len(Y)
for inter in range(self.__inter):
for i in range(N): ##进行N次放回式随机采样
row = np.random.randint(N)
self.__Xbar[inter,i,:] = X[row,:]
self.__Ybar[inter,i,:] = Y[row,:]
def train(self, lamb): ##进行训练
N = self.__Ybar.shape[1]
Lamb = np.eye(N) * lamb ##lambda矩阵
for inter in range(self.__inter): ##训练interation次
K = np.zeros((N,N)) ##初始化K矩阵 N×N
for i in range(N): ##计算K矩阵
for j in range(N):
#K[i,j] = exp(-gama * np.dot((X[i] - X[j]).T, (X[i] - X[j]))) ##RBF
K[i,j] = np.dot(self.__Xbar[inter,i], self.__Xbar[inter,j].T) ##linear
self.__beta[inter] = np.dot(np.linalg.inv(Lamb + K), self.__Ybar[inter]) ##根据公式计算训练参数beta
def predict(self, Xtest, Ytest): ##进行预测，Ein，Eout均可
py = np.zeros((len(Ytest), 1)) ##初始化预测结果数组
for inter in range(self.__inter):
Xmodel = self.__Xbar[inter] ##读入第inter次bootstrap采样数据
temp = np.zeros((len(Xmodel),1))
for i in range(len(Ytest)): #i--第i个测试数据
for j in range(len(Xmodel)): #j--第j个训练数据
temp[j] = np.dot(Xtest[i], Xmodel[j].T) #linear
#temp[j] = exp(-gama * np.dot((Xtest[i] - Xmodel[j]).T, (Xtest[i] - Xmodel[j]))) #temp--1～N训练数据与第i个数据求Kernel
py[i,:] = py[i,:] + [1 if np.dot(temp.T, self.__beta[inter])>=0 else -1] ##将interation个hypotheses预测的结果累加，针对第i个test数据
for i in range(len(py)): ##voting，累加interation个假设函数的预测值，大于0说明预测为+1的函数较多，小于0相反
if py[i, :] >= 0:
py[i, :] = 1
else:
py[i, :] = -1
accuracy = sum(py == Ytest)/len(Ytest) ##计算预测精度
return py, accuracy
def get_Xbar(self):
return self.__Xbar
def get_Ybar(self):
return self.__Ybar
def get_beta(self):
return self.__beta
X, Y = loaddata('hw2_lssvm_all.dat.txt')
Bag = Bagging(250, X[:400,:], Y[:400]) ##初始化Bootstrap对象
Bag.Bootstrap(X[:400,:], Y[:400]) ##进行Bootstrap采样
Bag.train(0.01) ##进行训练
py_in, acc_in = Bag.predict(X[:400,:], Y[:400]) ##预测Ein
py_out,acc_out = Bag.predict(X[400:,:], Y[400:]) ##预测Eout
##acc_in = array([0.68])
##acc_out = array([0.63636364])
##发现lambda对结果的影响较小