import numpy as np
import pandas as pd
from scipy.io import loadmat
import matplotlib.pyplot as plt
import scipy.optimize as opt
'''===================================函数部分=================================='''
'''数据可视化'''
def plot(X,y):
fig,ax = plt.subplots(figsize=(10,8))
ax.scatter(X, y, c='r', marker='x')
ax.set_xlabel('Change in water level(x)')
ax.set_ylabel('Water flowing out of the dam(y)')
plt.show()
'''代价函数,正则'''
def costReg(theta,X,y,lbd):
X = np.matrix(X)
y = np.matrix(y)
theta = np.matrix(theta)
cost = 1/(2*len(X)) * np.sum(np.power(([email protected]), 2))
reg = lbd/(2*len(X)) * np.sum(np.power(theta[:,1:], 2))
return cost + reg
'''梯度下降,正则'''
def gradientReg(theta,X,y,learning_rate):
theta = np.matrix(theta)
X = np.matrix(X)
y = np.matrix(y)
j = theta.shape[1] #2
sum_grad = np.zeros((1,j))
for i in range(j):
sum_grad[:,0] = (1 / len(X)) * np.sum((X @ theta.T - y).T * X[:,0])
reg = (learning_rate * theta[:, i]) / len(X)
sum_grad[:,i] = (1 / len(X)) * np.sum((X @ theta.T - y).T * X[:,i]) + reg
return sum_grad
'''计算预测值'''
def predict(Theta,X):
Theta = np.matrix(Theta)
y_predict = X * Theta.T
return y_predict
'''学习曲线函数training subset'''
def learning_curve(theta,Xv,yv,Xt,yt,n,lbd): # 2 <= n <= 11
error_train = np.zeros((n, 1)) #11
error_val = np.zeros((n, 1)) #11
for i in range(1,n+1): #1~11
Xtt = Xt[0:i, :] #2,3,4,5,6,7,8,9,10,11,12 注意,这里不要写Xt = Xt[0:i, :], python不知道把输入的Xt给谁
ytt = yt[0:i, :] #2,3,4,5,6,7,8,9,10,11,12
min = opt.minimize(fun=costReg, x0=theta, args=(Xtt,ytt,lbd), method='TNC', jac=gradientReg)
theta = min.x
# print('ddd',theta)
error_train[i-1,0] = costReg(theta,Xtt,ytt,0)
error_val[i-1,0] = costReg(theta,Xv,yv,0)
return error_train, error_val
'''特征变量扩展函数'''
def feature(X,p):
X_feature = np.zeros((X.shape[0],p))
for i in range(1,p+1):
X_feature[::,i-1] = np.power(X,i).flatten() #[::,i-1]指替换第i-1列,注意,替换的对象必须是(12,)格式,所以要拍平一下
return X_feature
'''标准化'''
def norm(X):
sigma = np.std(X, axis=0, ddof=1) #axis=0计算每一列的标准差,ddof=1表示自由度为1
mean = np.mean(X, axis=0)
norm = (X-mean)/sigma
return norm, mean, sigma
'''绘制lambda与error之间的关系'''
def lmd_error(lmd_lst,Xv,yv,Xt,yt):
error_t = np.zeros((len(lmd_lst),1))
error_v = np.zeros((len(lmd_lst),1))
for i in range(len(lmd_lst)):
min = opt.minimize(fun=costReg, x0=np.ones(p+1), args=(Xt,yt,lmd_lst[i]), method='TNC', jac=gradientReg)
theta2 = min.x
print(theta2)
error_t[i,0] = costReg(theta2,Xt,yt,0)
error_v[i,0] = costReg(theta2,Xv,yv,0)
return error_t,error_v
'''===================================计算部分=================================='''
'''导入数据'''
data = loadmat('D:\新大陆\吴恩达机器学习\ex5\ex5data1.mat')
X_raw = data['X']
y_raw = data['y']
print('X_raw:',X_raw.shape) #12,1
print('y_raw:',y_raw.shape) #12,1
# print(data)
'''数据可视化'''
# plot(X_raw,y_raw)
'''X_raw处理,加X0'''
X = np.insert(X_raw,0,np.ones(1),axis=1) #axis=1,插入一列,axis=0,插入一行
print('X:',X.shape)
'''创建初始theta'''
theta = np.ones(X.shape[1]) #特别注意
'''计算原始代价'''
print('original cost:',costReg(theta,X,y_raw,lbd=1))
'''计算原始梯度'''
print(gradientReg(theta,X,y_raw,learning_rate=1))
'''计算梯度下降有最小值时的theta'''
min = opt.minimize(fun=costReg, x0=theta, args=(X,y_raw,0), method='TNC', jac=gradientReg) #相比特征变量扩展以后的情况,这里的假设函数阶数太低了,用正则没什么效果。
Theta = min.x #array(2,)
'''画回归函数'''
A = np.linspace(start=X_raw.min(), stop=X_raw.max(), num=100) #array(100,)
AA = np.insert(np.reshape(A,(100,1)),0,np.ones(100),axis=1) #array(100,2)
B = predict(Theta,AA) #(100,1)
fix,ax = plt.subplots(figsize=(12,8))
ax.scatter(X_raw,y_raw,c='r',marker='x')
ax.plot(A,B,c='b')
ax.set_xlabel('Change in water level(x)')
ax.set_ylabel('Water flowing out of the dam(y)')
ax.set_title('Linear Fit')
# plt.show()
'''计算error_train & error_val并画学习曲线'''
X_val = data['Xval']
X_val_insert = np.insert(X_val,0,np.ones(X_val.shape[0]),axis=1) #array(21,2)
y_val = data['yval']
error_train, error_val = learning_curve(theta,X_val_insert,y_val,X,y_raw,11,0)
# print(error_train,error_val)
fix,ax = plt.subplots(figsize=(12,8))
ax.plot(np.linspace(start=2,stop=12,num=11),error_train,c='b',label='Train')
ax.plot(np.linspace(start=2,stop=12,num=11),error_val,c='g',label='Cross Validation')
ax.legend(loc=1)
ax.set_xlabel('Number of training examples')
ax.set_ylabel('Error')
ax.set_title('Learning curve for linear regression')
# plt.show()
'''将train test val 3个集子进行特征变量扩展'''
p = 6 #不知道为啥取6的时候才符合pdf上的图像,取8的时候稍微有点不一样
X_test = data['Xtest']
y_test = data['ytest']
X_train_exp = feature(X_raw,p) #12,p
# print(X_raw)
# print(X_train_exp.shape)
X_test_exp = feature(X_test,p) #12,p
# print(X_test_exp)
X_val_exp = feature(X_val,p) #12,p
# print(X_val_exp)
'''将数据进行norm处理'''
X_train_norm, X_train_mean, X_train_sigma = norm(X_train_exp) #12,p
# print(X_train_sigma.shape)
X_test_norm, aa, bb = norm(X_test_exp) #21,p
X_val_norm, cc, dd = norm(X_val_exp) #21,p
# print(X_val_norm.shape)
'''特征扩展后的函数图像'''
theta1 = np.zeros(p+1)
X_train_norm_insert = np.insert(X_train_norm,0,np.ones(X_train_norm.shape[0]),axis=1)
# print(X_train_norm_insert.shape)
#调整lmd为:0,0.001,0.003,0.1,0.3,1,3,10,100
lmd = 1
min = opt.minimize(fun=costReg, x0=theta1, args=(X_train_norm_insert,y_raw,lmd), method='TNC', jac=gradientReg)
#这里假设函数中theta数量就比较大了,需要有一个lbd来正则。这里写0,是因为需要看一下在不正则的情况下,过拟合是多么严重
theta1 = min.x
# print(theta1)
C = np.reshape(np.linspace(start=-80, stop=60, num=100), (100,1))
C_exp = feature(C, p)
C_norm = (C_exp - X_train_mean) / X_train_sigma #特别注意,仔细看这里的mean和sigma用的是谁的
# print(C_norm.shape)
C_insert = np.insert(C_norm,0,np.ones(C_norm.shape[0]),axis=1)
D = predict(theta1,C_insert)
fig,ax = plt.subplots(figsize=(12,8))
ax.plot(C,D,c='b', linestyle='dashed')
ax.scatter(X_raw,y_raw,c='r',marker='x')
ax.set_xlabel('Change in water level(x)')
ax.set_ylabel('Water flowing out of the dam(y)')
ax.set_title('Polynomial Regression Fit (lambda = 0.000000)')
plt.show()
'''特征扩展后的学习曲线'''
X_val_norm_insert = np.insert(X_val_norm,0,np.ones(X_val_norm.shape[0]),axis=1)
error_train2, error_val2 = learning_curve(theta1,X_val_norm_insert,y_val,X_train_norm_insert,y_raw,11,lmd)
fix,ax = plt.subplots(figsize=(12,8))
ax.plot(np.linspace(start=2,stop=12,num=11),error_train2,c='b',label='Train')
ax.plot(np.linspace(start=2,stop=12,num=11),error_val2,c='g',label='Cross Validation')
ax.legend(loc=1)
ax.set_xlabel('Number of training examples')
ax.set_ylabel('Error')
ax.set_title('Learning curve for linear regression')
plt.show()
'''绘制lambda与error之间的关系'''
lmd_lst = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10] #取11个,因为error_train2 和 error_val2内部正好各自有11个数
print(len(lmd_lst))
error_t,error_v = lmd_error(lmd_lst,X_val_norm_insert,y_val,X_train_norm_insert,y_raw)
print(error_v)
fix,ax = plt.subplots(figsize=(12,8))
ax.plot(np.linspace(start=0,stop=10,num=10),error_t,c='b',label='Train')
ax.plot(np.linspace(start=0,stop=10,num=10),error_v,c='g',label='Cross Validation')
ax.legend(loc=1)
ax.set_xlabel('lambda')
ax.set_ylabel('Error')
plt.show()
第一篇完整自己写的代码,
感谢所有机器学习道路上的引路人。