深度学习 第一课 KNN以及交叉验证

1 python numpy 常用函数

1.1 cPickle

Python标准库提供pickle和cPickle模块用于序列化。pickle模块中的两个主要函数是dump()和load()。

1.2 numpy

numpy.concatenate 连接两个矩阵 可以是按行连接或按列连接

numpy.tile(arr, (x, y)) 用于扩充数组 如果是数组扩充几倍 如果是举证扩充 (x, y) x行y列

numpy.bincount(a) 返回索引值在 a 出现的次数 比 a 中最大值大 1

2 KNN 详细实现以及交叉验证

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import cPickle as pickle 
import numpy as np 
import os
import random
import matplotlib.pyplot as plt

def load_CIFAR_batch(filename):
    with open(filename, 'r') as f: # rb
        datadict = pickle.load(f)

        X = datadict['data']
        Y = datadict['labels']
        X = X.reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).astype(float)
        Y = np.array(Y) # 转换成一个 labels 的数组

        # X 为 10000 * 3 * 32 * 32 的矩阵 
        return X, Y

def load_CIFAR1O(ROOT):
    print('start load_CIFAR1O')
    xs = []
    ys = []

    for b in range(1, 6):
        f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
        X, Y = load_CIFAR_batch(f)
        xs.append(X)
        ys.append(Y)
    Xtr = np.concatenate(xs)
    Ytr = np.concatenate(ys)
    del X, Y

    Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
    return Xtr, Ytr, Xte, Yte

class KNearestNeighbor:

    def __init__(self):
        pass

    def train(self, X, y):
        self.X_train = X # dataset
        self.y_train = y # label

    def predict(self, X, k=1):
        dists = self.compute_distances_two_loops(X)
        return self.predict_labels(dists, k=k)

    def compute_distances_no_loops(self, X):
        num_test = X.shape[0]
        num_train = self.X_train.shape[0]
        dists = np.zeros((num_test, num_train)) 

        """
        公式: (x - y) ^2 = x^2 + y^2 -2xy 
        1 - np.sum( (self.X_train)**2, axis=1) 
            计算训练集列方向的和 得到列方向的距离数组 即 y^2
        2 - np.tile( np.sum( (self.X_train)**2, axis=1), [num_test, 1]) 
            将 y^2 数组扩充成 [num_test, 1] 既 num_test 行 1 列的矩阵 例如： [1, 2, 3, 4, 5]
        3 - np.tile( np.sum( (X)**2, axis=1), [num_train, 1]).T
            计算 x^2 数组并且扩充成 [num_train, 1].T 的矩阵 例如: [1, 1, 1, 2, 2].T
            矩阵乘法是行乘以列这样来计算的所以需要转置
        4 - 计算 xy 矩阵相乘
        5 - 计算 x^2 + y^2 - 2xy 的值
        """ 
        train_2 = np.tile( np.sum( (self.X_train)**2, axis=1), [num_test, 1]) 
        test_2  = np.tile( np.sum( (X)**2, axis=1), [num_train, 1]).T
        test_train = X.dot(self.X_train.T)

        dists = train_2 + test_2 - 2*test_train
        return dists

    def predict_labels(self, dists, k=1):
        """
        返回预测的 labels
        """
        num_test = dists.shape[0]
        y_pred = np.zeros(num_test)
        for i in xrange(num_test):
            # 得到最近的k个点索引id 再得到这些 索引对应的 label
            # [:k] 表示从 [0:k]个点索引
            closest_y = []
            closest_idx = np.argsort(dists[i, :])[:k].tolist() 
            closest_y = self.y_train[closest_idx]

            # numpy.bincount 返回索引值在 closest_y 出现的次数 比 closest_y 中最大值大 1
            # 得到这k个点label出现的次数 选取出现次数最多的 label 为最好的结果
            counts = np.bincount(closest_y) 
            y_pred[i] = np.argmax(counts) 
        return y_pred

cifar10_dir = 'datasets/cifar-10-batches-py'
X_train, y_train, X_test, y_test = load_CIFAR1O(cifar10_dir)

# 数据库中有5w张图片 这里只使用了 5k
num_training = 5000
mask = range(num_training)
X_train = X_train[mask]
y_train = y_train[mask]

num_test = 500
mask = range(num_test)
X_test = X_test[mask]
y_test = y_test[mask]
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))

# 测试分类器
def test_classifier():
    classifier = KNearestNeighbor()
    classifier.train(X_train, y_train) # KNN 实际上没啥训练过程
    y_test_pred = classifier.predict(k=5) 

    # 看下准确率
    num_correct = np.sum(y_test_pred == y_test)
    accuracy = float(num_correct) / num_test
    print '%d个预测对了，总共%d个测试样本 => 准确率: %f' % (num_correct, num_test, accuracy)

# 交叉验证
def cost_validation():
    # K 折交叉验证 1.选取 n 个不同的 k 2.把数据分成 m 份 
    # 每一个 k 在这 m 分数据上做预测 取平均值即为在这个k对应的准确值 然后再选取最优的
    num_folds = 5 # 同一个k 5个结果取平均值
    k_choices = [1, 3, 5, 8, 10, 12, 15, 20, 50, 100]

    X_train_folds = []
    y_train_folds = []

    idxes = range(num_training)
    idx_folds = np.array_split(idxes, num_folds)
    for idx in idx_folds:
        X_train_folds.append( X_train[idx] )
        y_train_folds.append( y_train[idx] )
        
    k_to_accuracies = {}

    import sys
    classifier = KNearestNeighbor()
    Verbose = False
    for k in k_choices:
        if Verbose: print "processing k=%f" % k
        else: sys.stdout.write('.')

        k_to_accuracies[k] = list()
        for num in xrange(num_folds):
            if Verbose: print "processing fold#%i/%i" % (num, num_folds)
            
            X_cv_train = np.vstack( [ X_train_folds[x] for x in xrange(num_folds) if x != num ])
            y_cv_train = np.hstack( [ y_train_folds[x].T for x in xrange(num_folds) if x != num ])
            
            # 需要特别主要的是交叉验证过程中不能用测试集上的数据 所以这里是选了训练集上的某折数据来当测试集
            X_cv_test = X_train_folds[num]
            y_cv_test = y_train_folds[num]

            classifier.train(X_cv_train, y_cv_train)
            dists = classifier.compute_distances_no_loops(X_cv_test)
            y_cv_test_pred = classifier.predict_labels(dists, k=k)
            num_correct = np.sum(y_cv_test_pred == y_cv_test)
            k_to_accuracies[k].append( float(num_correct) / y_cv_test.shape[0] )

    # 输出k和对应的准确率
    for k in sorted(k_to_accuracies):
        for accuracy in k_to_accuracies[k]:
            print 'k = %d, accuracy = %f' % (k, accuracy)

    best_k = 6 # 根据上面交叉验证的结果，咱们确定最合适的k值为6，然后重新训练和测试一遍吧

3 为什么 KNN 没法用于实际生产中？

1.准确度不高

2.需要大量的实时计算和耗费空间 KNN 在训练期间实际上没做什么事情

4 做N折交叉验证

N折交叉验证

1.选取 n 个不同的 k 
2.把数据分成 m 份 
3.对于每个k 都可以做m次训练 每次训练的时候 
从这m中选一个当测试集而不能把预先的测试集拿来用 
从 [1, m] 依次选取当测试集 其他的当数据集来预测
每一个 k 在这 m 分数据上做预测 取平均值即为在这个k对应的准确值
然后再选取最优的K来得到最好的预测结果

交叉验证

交叉验证的结果