python实现支持向量机
最近学习了支持向量机，整理一下支持向量机的框架。
1.在logistic分类时，将训练集输入到模型中，训练集输入数据为{(x11,x12,…x1n),(x21,x22,…,x2n),…,(xm1,xm2,…,xmn)},
模型为f(xi) = sigmoid(xi*w.T+b)
若f(x)>0.5,则预测y‘=1，若f(x)<0.5,则预测y‘=0.
训练的损失函数为loss=（y-y'）^2
2.在支持向量机中与logistic相比，同样希望有一组权重能够将不同label的样本分开，但与logistic的不同在于支持向量机希望被分开的两组样本到平分线的距离能够尽可能的大。因此其目标函数变为
min 1/2 * ||w||^2
s.t. yi(w.T*xi+b) >= 1
加入拉格朗日乘子后，目标函数变为：
L（w,b,a）= 1/2||w||^2+∑ai(1-yi(w.T*xi+b))
= 1/2||w||^2 + 1/2∑∑ai*aj*yi*yj*xi.T*xj
其中
w = ∑ai*yi*xi
b = ∑ai*yi
3.针对非线性情况，这里使用核函数来将低维空间映射到高维空间，因此目标函数变为：
L（w,b,a）= ∑ai-1/2∑∑ai*aj*yi*yj*Kij
其中判别函数为
f(x) = ∑ai*yi*K(xi,x) + b
若f(x)>0,则y=1，否则y=-1
4.因此只需修改alpha和b即可，这里使用SMO来求解a
在SMO中，每次对两个a进行修改，因此首先选择两个a。在选择第一个a时对所有的a值进行循环，选择违反KKT原则最严重的a值。选择规则代码如下：
# 选择alpha序号
def SMO_get_alpha(self):
# 对所有alpha遍历
for i in range(self.sample_num):
# 选择违反KKT原则最严重的alpha
if 0 < self.alpha[i] < self.C:
if self.train_label[i] * self.fx(self.train_data[i]) != 1:
index2 = self.choose_another_alpha(i)
return i, index2
elif self.alpha[i] == 0:
if self.train_label[i] * self.fx(self.train_data[i]) < 1:
index2 = self.choose_another_alpha(i)
return i, index2
elif self.alpha[i] == self.C:
if self.train_label[i] * self.fx(self.train_data[i]) > 1:
index2 = self.choose_another_alpha(i)
return i, index2
return -1, -1
在选择第二个a值时，选择使得Ei-Ej最大的a。选择规则代码如下：
# 在已知第一个alpha的基础上，选择第二个alpha
def choose_another_alpha(self, index):
result_index = 0
temp_error = 0
for i in range(self.alpha.shape[0]):
diff_error = np.abs(self.error(index) - self.error(i))
temp = self.get_kernal( self.train_data[index],self.train_data[index]) + self.get_kernal(self.train_data[i],self.train_data[i]) - 2 * self.get_kernal(self.train_data[index],self.train_data[i])
if diff_error > temp_error and i != index and temp > 0:
temp_error = diff_error
result_index = i
return result_index
5.修改a值和b值：
首先修改a2，选择a2的取值范围，L为下限，H为上限：
当y1≠y2时，L=max(0, alpha2- alpha1）,H=min(C, C+ alpha2- alpha1 ）
当y1=y2时，L=max(0, alpha1+ alpha2）,H=min(C, alpha1+ alpha2-C）
对a2进行更新： 再对a1进行修改： 对b进行修改： 修改参数值代码： 6.整体代码： 7.试验结果：
若a2
若L
a1= a1 + y1 * y2 * (old_a2 - a2)
b1 = -e1 - y1 * K11 * (a1 - old_a1) - y2 * K12 * (a2 -old_a2) + b
b2 = -e2 - y1 * K12 * (a1 - old_a1) - y2 * K22 * (a2 -old_a2) + b
若 0 < a1 < C : b = b1
若 0 < a2 < C : b = b2
否则： b = (b1 + b2) / 2 def update(self, index1, index2):
# 计算运算量
old_alpha = self.alpha.copy()
x1 = self.train_data[index1]
y1 = self.train_label[index1]
x2 = self.train_data[index2]
y2 = self.train_label[index2]
K11 = self.get_kernal(x1,x1)
K22 = self.get_kernal(x2,x2)
K12 = self.get_kernal(x1,x2)
e1 = self.error(index1)
e2 = self.error(index2)
# 确定alpha范围
if y1 != y2:
L = max(0, old_alpha[index2] - old_alpha[index1])
H = min(self.C, self.C + old_alpha[index2] - old_alpha[index1])
else:
L = max(0, old_alpha[index1] + old_alpha[index2] - self.C)
H = min(self.C, old_alpha[index1] + old_alpha[index2])
# 运算alpha2
alpha2 = old_alpha[index2] + y2 * (e1 - e2) / (K11 + K22 - 2 * K12)
# 确定是否在范围内
if alpha2 < L:
alpha2 = L
elif alpha2 > H:
alpha2 = H
# 计算alpha1
alpha1 = old_alpha[index1] + y1 * y2 * (old_alpha[index2] - alpha2)
# 更新alpha
self.alpha[index1] = alpha1
self.alpha[index2] = alpha2
# 计算b
b1 = -e1 - y1 * K11 * (alpha1 - self.alpha[index1]) - y2 * K12 * (alpha2 - self.alpha[index2]) + self.bias
b2 = -e2 - y1 * K12 * (alpha1 - self.alpha[index1]) - y2 * K22 * (alpha2 - self.alpha[index2]) + self.bias
# 更新b
if 0 < alpha1 < self.C:
self.bias = b1
elif 0 < alpha2 < self.C:
self.bias = b2
else:
self.bias = (b1 + b2) / 2
# print(old_alpha - self.alpha)
print('E = {}'.format(np.linalg.norm(old_alpha - self.alpha)))
if np.linalg.norm(old_alpha - self.alpha) < self.e:
return True
else:
return False
import numpy as np
import matplotlib.pyplot as plt
#解决中文显示问题
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def get_data(d = 1):
data = np.zeros([1000,3])
temp1 = np.random.uniform(-13, 13, 500)
for i in range(500):
data[i][0] = temp1[i]
data[i][2] = -1
if data[i][0] < -7 or data[i][0] > 7:
data[i][1] = np.random.uniform(0,np.sqrt(13 * 13 - data[i][0] * data[i][0]),1)
else:
data[i][1] = np.random.uniform(np.sqrt(7 * 7 - data[i][0] * data[i][0]), np.sqrt(13 * 13 - data[i][0] * data[i][0]), 1)
temp2 = np.random.uniform(-3, 23, 500)
for i in range(500,1000):
data[i][0] = temp2[i-500]
data[i][2] = 1
if data[i][0] < 3 or data[i][0] > 17:
data[i][1] = np.random.uniform(-d - np.sqrt(13*13-(10 - data[i][0]) * (10 - data[i][0]) ), -d ,1)
else:
data[i][1] = np.random.uniform(-d - np.sqrt(13*13-(10 - data[i][0]) * (10 - data[i][0]) ), -d - np.sqrt(7*17-(10 - data[i][0]) * (10 - data[i][0]) ), 1)
np.random.shuffle(data)
origin_data = data.T[:2][:]
for i in range(len(data.T)):
data.T[i] -= np.mean(data.T[i])
data.T[i] /= np.std(data.T[i])
all_transpose_data = data.T[:2][:]
all_transpose_label = data.T[2][:]
return all_transpose_data.T,all_transpose_label.T,origin_data.T
class svm:
def __init__(self, train_data, test_data, train_label, test_label, origin_train_data,origin_test_data, C = 5, sigma = 0.5, e=0.01):
self.train_data = train_data
self.test_data = test_data
self.train_label = train_label
self.test_label = test_label
self.origin_train_data = origin_train_data
self.origin_test_data = origin_test_data
self.sample_num = len(train_data)
self.C = C
self.e = e
self.sigma = sigma
self.kernal = np.zeros([len(train_data), len(train_data)])
self.alpha = np.zeros(len(train_data))
self.bias = 0
# 获取核函数
def get_kernal(self,x1,x2):
return np.exp(np.linalg.norm(x1-x2)/(-2*self.sigma**2))
# 获取某一个样本对于所有训练样本的核函数向量
def get_kernal_i(self,x):
temp = np.zeros(self.sample_num)
for num in range(self.sample_num):
temp[num] = self.get_kernal(x,self.train_data[num])
return temp
# 计算f(x)
def fx(self, input):
result = np.sum(self.get_kernal_i(input) * self.train_label * self.alpha) + self.bias
return result
# 计算误差函数
def error(self,i):
return self.fx(train_data[i]) - self.train_label[i]
# 选择alpha序号
def SMO_get_alpha(self):
# 对所有alpha遍历
for i in range(self.sample_num):
# 选择违反KKT原则最严重的alpha
if 0 < self.alpha[i] < self.C:
if self.train_label[i] * self.fx(self.train_data[i]) != 1:
index2 = self.choose_another_alpha(i)
return i, index2
elif self.alpha[i] == 0:
if self.train_label[i] * self.fx(self.train_data[i]) < 1:
index2 = self.choose_another_alpha(i)
return i, index2
elif self.alpha[i] == self.C:
if self.train_label[i] * self.fx(self.train_data[i]) > 1:
index2 = self.choose_another_alpha(i)
return i, index2
return -1, -1
# 在已知第一个alpha的基础上，选择第二个alpha
def choose_another_alpha(self, index):
result_index = 0
temp_error = 0
for i in range(self.alpha.shape[0]):
diff_error = np.abs(self.error(index) - self.error(i))
temp = self.get_kernal( self.train_data[index],self.train_data[index]) + self.get_kernal(self.train_data[i],self.train_data[i]) - 2 * self.get_kernal(self.train_data[index],self.train_data[i])
if diff_error > temp_error and i != index and temp > 0:
temp_error = diff_error
result_index = i
return result_index
def update(self, index1, index2):
# 计算运算量
old_alpha = self.alpha.copy()
x1 = self.train_data[index1]
y1 = self.train_label[index1]
x2 = self.train_data[index2]
y2 = self.train_label[index2]
K11 = self.get_kernal(x1,x1)
K22 = self.get_kernal(x2,x2)
K12 = self.get_kernal(x1,x2)
e1 = self.error(index1)
e2 = self.error(index2)
# 确定alpha范围
if y1 != y2:
L = max(0, old_alpha[index2] - old_alpha[index1])
H = min(self.C, self.C + old_alpha[index2] - old_alpha[index1])
else:
L = max(0, old_alpha[index1] + old_alpha[index2] - self.C)
H = min(self.C, old_alpha[index1] + old_alpha[index2])
# 运算alpha2
alpha2 = old_alpha[index2] + y2 * (e1 - e2) / (K11 + K22 - 2 * K12)
# 确定是否在范围内
if alpha2 < L:
alpha2 = L
elif alpha2 > H:
alpha2 = H
# 计算alpha1
alpha1 = old_alpha[index1] + y1 * y2 * (old_alpha[index2] - alpha2)
# 更新alpha
self.alpha[index1] = alpha1
self.alpha[index2] = alpha2
# 计算b
b1 = -e1 - y1 * K11 * (alpha1 - self.alpha[index1]) - y2 * K12 * (alpha2 - self.alpha[index2]) + self.bias
b2 = -e2 - y1 * K12 * (alpha1 - self.alpha[index1]) - y2 * K22 * (alpha2 - self.alpha[index2]) + self.bias
# 更新b
if 0 < alpha1 < self.C:
self.bias = b1
elif 0 < alpha2 < self.C:
self.bias = b2
else:
self.bias = (b1 + b2) / 2
# print(old_alpha - self.alpha)
print('E = {}'.format(np.linalg.norm(old_alpha - self.alpha)))
if np.linalg.norm(old_alpha - self.alpha) < self.e:
return True
else:
return False
def train(self, train_num = 1000):
for i in range(train_num):
index1, index2 = self.SMO_get_alpha()
if index1 == -1:
print('结束迭代, iter = {}'.format(iter))
break
train_result = self.update(index1, index2)
if train_result == True:
print('结束迭代, iter = {}'.format(iter))
break
# 预测一次
def predict_once(self,x):
if self.fx(x) > 0:
return 1
else:
return -1
# 预测整个数据集
def predict(self,data):
result = np.array([self.predict_once(x) for x in data])
return result
# 获取测试集和训练集的准确率
def get_accuracy(self):
train_result = self.predict(self.train_data)
train_num = 0
for i in range(len(train_result)):
if train_result[i] == train_label[i]:
train_num += 1
train_accuracy = train_num / len(train_result)
print("训练集准确率为{}".format(train_accuracy))
test_result = self.predict(self.test_data)
print(train_result)
print(test_result)
test_num = 0
for i in range(len(test_result)):
if test_result[i] == test_label[i]:
test_num += 1
test_accuracy = test_num / len(test_result)
print("测试集准确率为{}".format(test_accuracy))
self.draw(test_result)
self.draw(self.test_label)
def draw(self,result):
class1 = np.zeros([1000,2])
class2 = np.zeros([1000,2])
class1_num = 0
class2_num = 0
for i in range(len(result)):
if result[i] == -1:
class1[class1_num] = self.origin_test_data[i]
class1_num += 1
else:
class2[class2_num] = self.origin_test_data[i]
class2_num += 1
colors1 = '#00CED1' # 点的颜色
colors2 = '#DC143C'
plt.scatter(class1.T[0],class1.T[1],c=colors1,label = "上半园")
plt.scatter(class2.T[0], class2.T[1], c=colors2, label="下半园")
plt.legend()
plt.show()
if __name__ == "__main__":
train_data,train_label,origin_train_data = get_data(0)
test_data, test_label,origin_test_data = get_data(0)
my_svm = svm(train_data,test_data,train_label,test_label,origin_train_data,origin_test_data)
my_svm.train(1000)
my_svm.get_accuracy()
问题：
以上代码为自动生成双月型样本点，并对其训练，使用的是高斯核。其训练集和测试集均为100%。
图形化效果如下：
d = 1，线性可分
d = 0，近似线性可分
d = -4， 线性不可分
