1 概述
基础的理论知识参考线性SVM与Softmax分类器。
代码实现环境：python3
2 数据处理
2.1 加载数据集
将原始数据集放入“data/cifar10/”文件夹下。
-
### 加载cifar10数据集
-
-
import os
-
import pickle
-
import random
-
import numpy as np
-
import matplotlib.pyplot as plt
-
-
def load_CIFAR_batch(filename):
-
"""
-
cifar-10数据集是分batch存储的，这是载入单个batch
-
-
@参数 filename: cifar文件名
-
@r返回值: X, Y: cifar batch中的 data 和 labels
-
"""
-
-
with open(filename,'rb') as f:
-
datadict=pickle.load(f,encoding='bytes')
-
-
X=datadict[b'data']
-
Y=datadict[b'labels']
-
-
X=X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
-
Y=np.array(Y)
-
-
return X, Y
-
-
def load_CIFAR10(ROOT):
-
"""
-
读取载入整个 CIFAR-10 数据集
-
-
@参数 ROOT: 根目录名
-
@return: X_train, Y_train: 训练集 data 和 labels
-
X_test, Y_test: 测试集 data 和 labels
-
"""
-
-
xs=[]
-
ys=[]
-
-
for b in range(1,6):
-
f=os.path.join(ROOT, "data_batch_%d" % (b, ))
-
X, Y=load_CIFAR_batch(f)
-
xs.append(X)
-
ys.append(Y)
-
-
X_train=np.concatenate(xs)
-
Y_train=np.concatenate(ys)
-
-
del X, Y
-
-
X_test, Y_test=load_CIFAR_batch(os.path.join(ROOT, "test_batch"))
-
-
return X_train, Y_train, X_test, Y_test
-
-
X_train, y_train, X_test, y_test = load_CIFAR10('data/cifar10/')
-
-
print(X_train.shape)
-
print(y_train.shape)
-
print(X_test.shape)
-
print( y_test.shape)
运行结果如下：
-
(50000, 32, 32, 3)
-
(50000,)
-
(10000, 32, 32, 3)
-
(10000,)
2.2 划分数据集
将加载好的数据集划分为训练集，验证集，以及测试集。
-
## 划分训练集，验证集，测试集
-
-
num_train = 49000
-
num_val = 1000
-
num_test = 1000
-
-
# Validation set
-
mask = range(num_train, num_train + num_val)
-
X_val = X_train[mask]
-
y_val = y_train[mask]
-
-
# Train set
-
mask = range(num_train)
-
X_train = X_train[mask]
-
y_train = y_train[mask]
-
-
# Test set
-
mask = range(num_test)
-
X_test = X_test[mask]
-
y_test = y_test[mask]
-
-
print('Train data shape: ', X_train.shape)
-
print('Train labels shape: ', y_train.shape)
-
print('Validation data shape: ', X_val.shape)
-
print('Validation labels shape ', y_val.shape)
-
print('Test data shape: ', X_test.shape)
-
print('Test labels shape: ', y_test.shape)
运行结果为：
-
Train data shape: (49000, 3072)
-
Validation data shape: (1000, 3072)
-
Test data shape: (1000, 3072)
2.3 去均值归一化
将划分好的数据集归一化，即：所有划分好的数据集减去均值图像。
-
# Processing: subtract the mean images
-
mean_image = np.mean(X_train, axis=0)
-
-
X_train -= mean_image
-
X_val -= mean_image
-
X_test -= mean_image
-
-
# append the bias dimension of ones (i.e. bias trick)
-
X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])#堆叠数组
-
X_val = np.hstack([X_val, np.ones((X_val.shape[0], 1))])
-
X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])
-
print('Train data shape: ', X_train.shape)
-
print('Validation data shape: ', X_val.shape)
-
print('Test data shape: ', X_test.shape)
运行结果为：
-
Train data shape: (49000, 3073)
-
Validation data shape: (1000, 3073)
-
Test data shape: (1000, 3073)
3 线性SVM分类器
3.1 定义线性SVM分类器
关键的是线性SVM的梯度推导过程。具体的可以看看这篇文章。
-
#Define a linear SVM classifier
-
-
class LinearSVM(object):
-
""" A subclass that uses the Multiclass SVM loss function """
-
def __init__(self):
-
self.W = None
-
-
def loss_vectorized(self, X, y, reg):
-
"""
-
Structured SVM loss function, naive implementation (with loops).
-
Inputs:
-
- X: A numpy array of shape (num_train, D) contain the training data
-
consisting of num_train samples each of dimension D
-
- y: A numpy array of shape (num_train,) contain the training labels,
-
where y[i] is the label of X[i]
-
- reg: (float) regularization strength
-
Outputs:
-
- loss: the loss value between predict value and ground truth
-
- dW: gradient of W
-
"""
-
-
# Initialize loss and dW
-
loss = 0.0
-
dW = np.zeros(self.W.shape)
-
-
# Compute the loss
-
num_train = X.shape[0]
-
scores = np.dot(X, self.W)
-
correct_score = scores[range(num_train), list(y)].reshape(-1, 1)
-
margin = np.maximum(0, scores - correct_score + 1) # delta = 1
-
margin[range(num_train), list(y)] = 0 #分对的损失为0
-
loss = np.sum(margin) / num_train + 0.5 * reg * np.sum(self.W * self.W) #reg就是权重lamda
-
-
# Compute the dW
-
num_classes = self.W.shape[1]
-
mask = np.zeros((num_train, num_classes))
-
mask[margin > 0] = 1
-
mask[range(num_train), list(y)] = 0
-
mask[range(num_train), list(y)] = -np.sum(mask, axis=1)
-
dW = np.dot(X.T, mask)
-
dW = dW / num_train + reg * self.W
-
-
return loss, dW
-
-
def train(self, X, y, learning_rate = 1e-3, reg = 1e-5, num_iters = 100,
-
batch_size = 200, print_flag = False):
-
"""
-
Train linear SVM classifier using SGD
-
Inputs:
-
- X: A numpy array of shape (num_train, D) contain the training data
-
consisting of num_train samples each of dimension D
-
- y: A numpy array of shape (num_train,) contain the training labels,
-
where y[i] is the label of X[i], y[i] = c, 0 <= c <= C
-
- learning rate: (float) learning rate for optimization
-
- reg: (float) regularization strength
-
- num_iters: (integer) numbers of steps to take when optimization
-
- batch_size: (integer) number of training examples to use at each step
-
- print_flag: (boolean) If true, print the progress during optimization
-
Outputs:
-
- loss_history: A list containing the loss at each training iteration
-
"""
-
-
loss_history = []
-
num_train = X.shape[0]
-
dim = X.shape[1]
-
num_classes = np.max(y) + 1
-
-
# Initialize W
-
if self.W == None:
-
self.W = 0.001 * np.random.randn(dim, num_classes)
-
-
# iteration and optimization
-
for t in range(num_iters):
-
idx_batch = np.random.choice(num_train, batch_size, replace=True)
-
X_batch = X[idx_batch]
-
y_batch = y[idx_batch]
-
loss, dW = self.loss_vectorized(X_batch, y_batch, reg)
-
loss_history.append(loss)
-
self.W += -learning_rate * dW
-
-
if print_flag and t%100 == 0:
-
print('iteration %d / %d: loss %f' % (t, num_iters, loss))
-
-
return loss_history
-
-
def predict(self, X):
-
"""
-
Use the trained weights of linear SVM to predict data labels
-
Inputs:
-
- X: A numpy array of shape (num_train, D) contain the training data
-
Outputs:
-
- y_pred: A numpy array, predicted labels for the data in X
-
"""
-
-
y_pred = np.zeros(X.shape[0])
-
scores = np.dot(X, self.W)
-
y_pred = np.argmax(scores, axis=1)
-
-
return y_pred
3.2 无交叉验证
3.2.1 训练模型
-
##Stochastic Gradient Descent
-
-
svm = LinearSVM()
-
loss_history = svm.train(X_train, y_train, learning_rate = 1e-7, reg = 2.5e4, num_iters = 2000,
-
batch_size = 200, print_flag = True)
运行结果如下：
-
iteration 0 / 2000: loss 407.076351
-
iteration 100 / 2000: loss 241.030820
-
iteration 200 / 2000: loss 147.135737
-
iteration 300 / 2000: loss 90.274781
-
iteration 400 / 2000: loss 56.509895
-
iteration 500 / 2000: loss 36.654007
-
iteration 600 / 2000: loss 23.732160
-
iteration 700 / 2000: loss 16.340341
-
iteration 800 / 2000: loss 11.538806
-
iteration 900 / 2000: loss 9.482515
-
iteration 1000 / 2000: loss 7.414343
-
iteration 1100 / 2000: loss 6.240377
-
iteration 1200 / 2000: loss 5.774960
-
iteration 1300 / 2000: loss 5.569365
-
iteration 1400 / 2000: loss 5.326023
-
iteration 1500 / 2000: loss 5.708757
-
iteration 1600 / 2000: loss 4.731255
-
iteration 1700 / 2000: loss 5.516500
-
iteration 1800 / 2000: loss 4.959480
-
iteration 1900 / 2000: loss 5.447249
3.2.2 预测
-
# Use svm to predict
-
# Training set
-
y_pred = svm.predict(X_train)
-
num_correct = np.sum(y_pred == y_train)
-
accuracy = np.mean(y_pred == y_train)
-
print('Training correct %d/%d: The accuracy is %f' % (num_correct, X_train.shape[0], accuracy))
-
-
# Test set
-
y_pred = svm.predict(X_test)
-
num_correct = np.sum(y_pred == y_test)
-
accuracy = np.mean(y_pred == y_test)
-
print('Test correct %d/%d: The accuracy is %f' % (num_correct, X_test.shape[0], accuracy))
运行结果如下：
-
Training correct 18799/49000: The accuracy is 0.383653
-
Test correct 386/1000: The accuracy is 0.386000
3.3 有交叉验证
3.3.1 训练模型
-
#Cross-validation
-
-
learning_rates = [1.4e-7, 1.5e-7, 1.6e-7]
-
regularization_strengths = [8000.0, 9000.0, 10000.0, 11000.0, 18000.0, 19000.0, 20000.0, 21000.0]
-
-
results = {}
-
best_lr = None
-
best_reg = None
-
best_val = -1 # The highest validation accuracy that we have seen so far.
-
best_svm = None # The LinearSVM object that achieved the highest validation rate.
-
-
for lr in learning_rates:
-
for reg in regularization_strengths:
-
svm = LinearSVM()
-
loss_history = svm.train(X_train, y_train, learning_rate = lr, reg = reg, num_iters = 2000)
-
y_train_pred = svm.predict(X_train)
-
accuracy_train = np.mean(y_train_pred == y_train)
-
y_val_pred = svm.predict(X_val)
-
accuracy_val = np.mean(y_val_pred == y_val)
-
if accuracy_val > best_val:
-
best_lr = lr
-
best_reg = reg
-
best_val = accuracy_val
-
best_svm = svm
-
results[(lr, reg)] = accuracy_train, accuracy_val
-
print('lr: %e reg: %e train accuracy: %f val accuracy: %f' %
-
(lr, reg, results[(lr, reg)][0], results[(lr, reg)][1]))
-
print('Best validation accuracy during cross-validation:\nlr = %e, reg = %e, best_val = %f' %
-
(best_lr, best_reg, best_val))
3.3.2 预测
-
# Use the best svm to test
-
y_test_pred = best_svm.predict(X_test)
-
num_correct = np.sum(y_test_pred == y_test)
-
accuracy = np.mean(y_test_pred == y_test)
-
print('Test correct %d/%d: The accuracy is %f' % (num_correct, X_test.shape[0], accuracy))
运行结果为：
Test correct 372/1000: The accuracy is 0.372000