深度学习入门（上）-第三章 案例实战

15.python环境搭建

16.Eclipse搭建python环境
安装Eclipse，安装JDK。
安装PyDev模块（帮助-安装新软件），配置Python环境（窗口-首选项）。
17.动手完成简单神经网络
完成3层的神经网络。

import numpy as np
def sigmoid(x, deriv = False):
    if (deriv == True):
        return x*(1-x)
    return 1/(1+np.exp(-x))
# data
x = np.array([[0,0,1],
            [0,1,1],
            [1,0,1],
            [1,1,1],
            [0,0,1]]
)
#查看数据维度,可知有5个样本，每个样本有3点特征
print (x.shape)  #（5，3）
# label
y = np.array([[0],
            [1],
            [1],
            [0],
            [0]]
)
print (y.shape)  #（5，1）
np.random.seed(1)
# 定义3层神经网络，要明确权值w的大小
w0 = 2*np.random.random((3,4)) -1 #w0前连3个特征，后连4个神经元
w1 = 2*np.random.random((4,1)) -1 #w1前连4个神经元，后连1个神经元（只有一个输出值（0||1））
print (w0)    #2*x-1后可见，值[-1,+1]
# 进行神经网络的构造以及迭代计算
# python 2.7 xrange
# python 3 range
for j in range(60000):
    # 前向传播
    l0 = x
    l1 = sigmoid(np.dot(l0,w0))
    l2 = sigmoid(np.dot(l1,w1))  #预测结果
    # 将预测结果与真实值y进行比较，loss的构造
    l2_error = y - l2
    if (j%10000) == 0:
        print('Error'+ str(np.mean(np.abs(l2_error))))
    #进行反向传播的求导操作，l2_error相当于权重系数
    l2_delta = l2_error * sigmoid(l2,deriv = True)
    l1_error = l2_delta.dot(w1.T)
    l1_delta = l1_error * sigmoid(l1,deriv = True)
    # 权值更新
    w1 += l1.T.dot(l2_delta)  #不光要把上一层的错误传下来，还要算自身的梯度。
    w0 += l0.T.dot(l1_delta)

---

18.感受神经网络的强大
对比神经网络模型和softmax分类器（线性分类）模型在线性不可分的数据集上的效果。神经网络胜出。
区别：用softmax分类器不像神经网络是一种层次结构，还有一些激活函数等等，只是通过一种简单的结构来求解分类任务。
制作数据drawData.py

import numpy as np
import matplotlib.pyplot as plt

#ubuntu 16.04 sudo pip instal matplotlib

plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

np.random.seed(0)
N = 100 # number of points per class
D = 2 # dimensionality
K = 3 # number of classes
X = np.zeros((N*K,D))
y = np.zeros(N*K, dtype='uint8')
for j in xrange(K):
  ix = range(N*j,N*(j+1))
  r = np.linspace(0.0,1,N) # radius
  t = np.linspace(j*4,(j+1)*4,N) + np.random.randn(N)*0.2 # theta
  X[ix] = np.c_[r*np.sin(t), r*np.cos(t)]
  y[ix] = j
fig = plt.figure()
plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)
plt.xlim([-1,1])
plt.ylim([-1,1])
plt.show()

实验效果

用softmax分类器来拟合数据lineCla.py

#Train a Linear Classifier
import numpy as np
import matplotlib.pyplot as plt


np.random.seed(0)
N = 100 # number of points per class
D = 2 # dimensionality
K = 3 # number of classes
X = np.zeros((N*K,D))
y = np.zeros(N*K, dtype='uint8')
for j in xrange(K):
  ix = range(N*j,N*(j+1))
  r = np.linspace(0.0,1,N) # radius
  t = np.linspace(j*4,(j+1)*4,N) + np.random.randn(N)*0.2 # theta
  X[ix] = np.c_[r*np.sin(t), r*np.cos(t)]
  y[ix] = j



W = 0.01 * np.random.randn(D,K)
b = np.zeros((1,K))

# some hyperparameters
step_size = 1e-0
reg = 1e-3 # regularization strength

# gradient descent loop
num_examples = X.shape[0]
for i in xrange(1000):
  #print X.shape
  # evaluate class scores, [N x K]
  scores = np.dot(X, W) + b   #x:300*2 scores:300*3
  #print scores.shape 
  # compute the class probabilities
  exp_scores = np.exp(scores)
  probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True) # [N x K] probs:300*3
  print probs.shape 
  # compute the loss: average cross-entropy loss and regularization
  corect_logprobs = -np.log(probs[range(num_examples),y]) #corect_logprobs:300*1
  print corect_logprobs.shape
  data_loss = np.sum(corect_logprobs)/num_examples
  reg_loss = 0.5*reg*np.sum(W*W)
  loss = data_loss + reg_loss
  if i % 100 == 0:
    print "iteration %d: loss %f" % (i, loss)

  # compute the gradient on scores
  dscores = probs
  dscores[range(num_examples),y] -= 1
  dscores /= num_examples

  # backpropate the gradient to the parameters (W,b)
  dW = np.dot(X.T, dscores)
  db = np.sum(dscores, axis=0, keepdims=True)

  dW += reg*W # regularization gradient

  # perform a parameter update
  W += -step_size * dW
  b += -step_size * db
  scores = np.dot(X, W) + b
predicted_class = np.argmax(scores, axis=1)
print 'training accuracy: %.2f' % (np.mean(predicted_class == y))

h = 0.02
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))
Z = np.dot(np.c_[xx.ravel(), yy.ravel()], W) + b
Z = np.argmax(Z, axis=1)
Z = Z.reshape(xx.shape)
fig = plt.figure()
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral, alpha=0.8)
plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.show()

实验效果：

用神经网络拟合数据NNCla.py

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(0)
N = 100 # number of points per class
D = 2 # dimensionality
K = 3 # number of classes
X = np.zeros((N*K,D))
y = np.zeros(N*K, dtype='uint8')
for j in xrange(K):
  ix = range(N*j,N*(j+1))
  r = np.linspace(0.0,1,N) # radius
  t = np.linspace(j*4,(j+1)*4,N) + np.random.randn(N)*0.2 # theta
  X[ix] = np.c_[r*np.sin(t), r*np.cos(t)]
  y[ix] = j

h = 100 # size of hidden layer
W = 0.01 * np.random.randn(D,h)# x:300*2  2*100
b = np.zeros((1,h))
W2 = 0.01 * np.random.randn(h,K)
b2 = np.zeros((1,K))

# some hyperparameters
step_size = 1e-0
reg = 1e-3 # regularization strength

# gradient descent loop
num_examples = X.shape[0]
for i in xrange(2000):

  # evaluate class scores, [N x K]
  hidden_layer = np.maximum(0, np.dot(X, W) + b) # note, ReLU activation hidden_layer:300*100
  #print hidden_layer.shape
  scores = np.dot(hidden_layer, W2) + b2  #scores:300*3
  #print scores.shape
  # compute the class probabilities
  exp_scores = np.exp(scores)
  probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True) # [N x K]
  #print probs.shape

  # compute the loss: average cross-entropy loss and regularization
  corect_logprobs = -np.log(probs[range(num_examples),y])
  data_loss = np.sum(corect_logprobs)/num_examples
  reg_loss = 0.5*reg*np.sum(W*W) + 0.5*reg*np.sum(W2*W2)
  loss = data_loss + reg_loss
  if i % 100 == 0:
    print "iteration %d: loss %f" % (i, loss)

  # compute the gradient on scores
  dscores = probs
  dscores[range(num_examples),y] -= 1
  dscores /= num_examples

  # backpropate the gradient to the parameters
  # first backprop into parameters W2 and b2
  dW2 = np.dot(hidden_layer.T, dscores)
  db2 = np.sum(dscores, axis=0, keepdims=True)
  # next backprop into hidden layer
  dhidden = np.dot(dscores, W2.T)
  # backprop the ReLU non-linearity
  dhidden[hidden_layer <= 0] = 0
  # finally into W,b
  dW = np.dot(X.T, dhidden)
  db = np.sum(dhidden, axis=0, keepdims=True)

  # add regularization gradient contribution
  dW2 += reg * W2
  dW += reg * W

  # perform a parameter update
  W += -step_size * dW
  b += -step_size * db
  W2 += -step_size * dW2
  b2 += -step_size * db2
hidden_layer = np.maximum(0, np.dot(X, W) + b)
scores = np.dot(hidden_layer, W2) + b2
predicted_class = np.argmax(scores, axis=1)
print 'training accuracy: %.2f' % (np.mean(predicted_class == y))


h = 0.02
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))
Z = np.dot(np.maximum(0, np.dot(np.c_[xx.ravel(), yy.ravel()], W) + b), W2) + b2
Z = np.argmax(Z, axis=1)
Z = Z.reshape(xx.shape)
fig = plt.figure()
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral, alpha=0.8)
plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.show()

实验效果：

19.神经网络案例-cifar分类任务
让神经网络学习每个类别有什么特征。对于新的数据，用神经网络进行分类。
步骤：

• 数据切分：训练集，验证集，测试集
• 搭建神经网络的结构：分模块构造神经网络
• 训练神经网络完成分类任务
  主体代码：
  fc_net.py

# fc_net.py #网络的整体架构
from layer_utils import *
import numpy as np
class TwoLayerNet(object):

    def __init__(self, input_dim = 3*32*32, hidden_dim = 100, num_classes = 10,
                 weight_scale=1e-3, reg=0.0):
        '''
        initialize a new network
        Inputs:
            dropout: Scalar between 0 and 1 giving dropout strength.
            weight_scale: Scalar giving the standard deviation for random 
                          initialization of the weights.(We hope weights are not so large)
            reg: Scalar giving L2 regularization strength.
        '''
        self.params = []
        self.reg = reg
        self.params['w1'] = weight_scale * np.random.randn(input_dim, hidden_dim)
        sels.params['b1] =np.zeros((1, hidden_dim))
        self.params['w2'] = weight_scale * np.random.randn(hidden_dim， num_classes)
        sels.params['b2] =np.zeros((1, num_classes))

    def loss(self, X, y=None):
        '''
        compute loss and gradient for a minibatch of data
        前向传播
        '''
        scores = None
        N = X.shape[0]
        W1, b1 = self.params['w1'], self.params['b1']
        W2, b2 = self.params['w2'], self.params['b2']
        h1, cache1 = affine_relu_forward(X, W1, b1)
        out, cache2 = affine_forward(h1, W2, b2)
        scores = out

        if y is None:
            return scores

        loss, grads = 0, {}
        data_loss, dscores = softmax_loss(scores, y)    
        reg_loss = 0.5 * self.reg * np.sum(W1*W1) + 0.5*self.reg * np.sum(W2*W2)
        loss = data_loss + reg_loss

        # Backward pass:compute gradients
        dh1, dW2, db2 = affine_backward(dscores, cache2)
        dX, dW1, db1 = affine_relu_backward(dh1, cache1)
        # Add the regularization gradient contribution
        dW2 += self.reg * W2   #self.reg是正则化惩罚力度
        dW1 += self.reg * W1
        grads['W1'] = dW1
        grads['b1'] = db1
        grads['W2'] = dW2
        grads['b2'] = db2

        return loss, grads

layer_utils.py

# layer_utils.py
from layers import *

def affine_relu_forward(x, w, b):
    '''
    Returns a tuple of:
        -out:output from the Relu
        -cache:object to give to the backward pass
    '''

    a, fc_cache = affine_forward(x,w,b)
    out, relu_cache = relu_forward(a)
    cache = (fc_cache, relu_cache)
    return out, cache

def affine_relu_backward(dout, cache):
    fc_cache, relu_cache = cache
    da = relu_backward(dout, relu_cache)
    dx, dw, db = affine_backward(da, fc_cache)
    return dx, dw, db

layers.py

# layers.py
import numpy as np

def affine_forward(x, w, b):
    out = None

    N = x.shape[0]
    x_row = x.reshape(N, -1)
    out = np.dot(x_row, w) + b
    cache = (x, w, b)

    return out, cache

def affine_backward(dout, cache):
    x, w, b = cache
    dx, dw, db = None, None, None
    dx = np.dot(dout, w.T)
    dx = np.reshape(dx, x.shape)
    x_row = x.reshape(x.shape[0], -1)
    dw = np.dot(x_row.T, dout)
    db = np.sum(dout, axis=0, keepdims=True)

    return dx, dw, db

def relu_forward(x):

    out = None
    out = ReLU(x)
    cache = x

    return out, cache

def relu_backward(dout, cache):
    dx, x = None, cache
    dx = dout
    dx[x <= 0]= 0

    return dx

def softmax_loss(x, y):
    # data normalization
    probs = np.exp(x - np.max(x, axis=1, keepdims=True))
    probs /= np.sum(probs, axis=1, keepdims=True)
    N = x.shape[0]
    loss = -np.sum(np.log(probs[np.arange(N), y])) / N
    dx = probs.copy()
    dx[np.arange(N), y] -= 1
    dx /= N

    return loss, dx

def ReLU(x):
    ''' ReLU non-linearity'''
    return np.maximum(0, x)

训练数据集的启动入口
two_layer_fc_net_start.py

# two_layer_fc_net_start.py
import matplotlib.pyplot as plt
from fc_net import *
from data_utils import get_CIFAR10_data
from solver import Solver

data  = get_CIFAR10_data()
model = TwoLayerNet(reg = 0.9)
solver = Solver(model, data, 
                lr_decay=0.95,
                print_every=100, num_epochs=40, batch_size=400,
                update_rule='sgd_momentum',
                optim_config={'learning_rate': 5e-4, 'momentum':0.9})

solver.train()

关于Cifar10分类的完整代码，请参看：
https://github.com/sunshinezhihuo/cifar10

Solver解释：
用Solver制定网络的超参数。
衰减策略：学习率衰减self.lr_decay，针对于epoch而言的
self.batch_size：一次训练多少张图片
self.num_epochs：迭代多少轮，一轮是全部data.
前向传播要loss，反向传播要w1,d1,w2,d2，之后进行权重参数的更新，这样，就完成了一次网络迭代的过程。
网络的又快又好发展
快：momentum：动量，主要用在权重更新的时候，是用来修改检索方向加快收敛速度的一种简单方法。
即，momentum可以加速收敛，方便网络快速学习。
好：learning rate decay

关于深度学习超参数简单理解，请参见https://zhuanlan.zhihu.com/p/23906526