【TensorFlow】官方MNIST数据集神经网络实例详解

这几天开始学tensorflow训练神经网络，先来做一下学习记录。
一周目，用的书是Tensorflow+实战Google深度学习框架，第五章5.2.1的训练神经网络，发现代码存在一定的问题，MNIST手写体数字识别问题，前向传播神经网络，使用带指数衰减的学习率设置、使用正则化来避免过拟合，以及使用华东平均模型来使最终模型更加健壮。
直接使用read_data_sets()无法自动下载mnist数据集，可能是需要科学上网，也有博主说官网已经下载不了了。我是自己在极客学院直接手动下载的MNIST数据集，然后放在.py文件下的同目录下，数据集下载地址：
http://wiki.jikexueyuan.com/project/tensorflow-zh/tutorials/mnist_download.html

代码我看很多改的都不太行，就放了一个自己的改进版本，亲测有效。

#学习《TensorFlow实战Google深度学习框架》
#!/usr/bin/env python 
# -*- coding:utf-8 -*-
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

#mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
#MNIST数据集的相关的常数
#输入层的节点数，对于MNIST数据集，这个等于图片的像素
INPUT_NODE = 784
#输出层的节点数，这个等于类别的数目。因为在MNIST数据集中需要区分的是0-9这10个数字。
OUTPUT_NODE = 10
#设置神经网络的参数
#隐藏层节点数，这里只使用一个隐藏层的网络结构作为样例。这个隐藏层有500个节点。
LAYER1_NODE = 500
#一个训练batch中的训练数据个数。数字越小时，训练过程接近随机梯度下降；数据越大时，训练接近梯度下降
BATCH_SIZE = 100

#基础的学习率
LEARNING_RATE_BASE = 0.8
#学习率的衰减率
LEARNING_RATE_DECAY = 0.99
#描述模型复杂度的正则化项在损失函数中的系数
REGULARIZATION_RATE = 0.0001
#训练次数
TRAINING_STEPS = 30000
#滑动平均衰减
MOVING_AVERAGE_DECAY = 0.99

#一个辅助函数，给定神经网络的输入和所有参数，计算神经网络的前向传播结果。在这里定义一个ReLU激活函数的三层全链接神经网络。
#通过加入隐藏层实现多层网络结构，通过ReLU激活函数实现去线性化。在这个函数中也支持传入用于计算参数均值的类。这样方便在测试时使用滑动平均模型
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
    #当没有提供滑动平均类是，直接使用参数当前的取值
    if avg_class == None:
    	#计算隐藏层的前向传播结果，这里使用了ReLU激活函数
        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
        #计算输出层的前向传播结果，因为在计算损失函数时会一并计算softmax函数，所以这里不需要加入激活函数。而且不加入softmax不会影响预测结果。
        #因为预测时使用的是不用于对应节点输出值的相对大小，有没有softmax层对最后的分类结果的计算没有影响。于是在计算整个神经网络的前向传播时
        #可以不加最后的softmax层。
        return tf.matmul(layer1, weights2) + biases2
   #否则，使用滑动平均值
    else:
       #首先使用avg_class.average函数来计算得出变量的滑动平均值。
        #然后再计算相应的神经网络前向传播的结果。
        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

#定义训练过程
def train(mnist):
   #占位符，定义x，y_变量
    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name = 'x-input')
    y_ = tf.placeholder(tf.int32, [None, OUTPUT_NODE], name = 'y-input')
    #定义存储训练的变量。这个变量不需要计算滑动平均，所以这里指定这个变量为不可训练的变量（trainable = False）。在使用TensorFlow训练神经网络时，
    #一般会将代表训练轮数的变量指定为不可训练的参数
    global_step = tf.Variable(0, trainable=False)
    #生成隐藏层的参数
    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
    #生成输出层的参数
    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))
    #计算在当前参数下神经网络前向传播的结果。这里给出的用于计算滑动平均的类为None，所以函数不会使用参数滑动平均
    y = inference(x, None, weights1, biases1, weights2, biases2)
    #给定滑动平均衰减率和训练轮数的变量，初始化滑动平均类。
    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
    #在所有代表神经网络参数的变量上使用滑动平均，其他的辅助变量(比如global_step)就不需要了。tf.trainable_variables返回的就是图像
    #GraphKeys.TRAINABLE_VARIABLES中的元素。这个集合的元素就是所有没有指定trainable = False的参数
    variables_averages_op = variable_averages.apply(tf.trainable_variables())
    #计算使用了滑动平均之后的前向传播的结果，滑动平均不会改变变量本身的取值，而是会维护一个影子变量来记录其滑动平均值。所以当需要使用这个滑动
    #平均值时，需要明确调用average函数
    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)
    #计算交叉熵作为刻画预测值和真实值之间差距的损失函数。函数第一个参数是神经网络不包括softmax层的前向传播结果，第二层是训练数据的正确答案。
    #因为答案是一个长度为10的数组，而该函数是需要提供一个正确答案的数字，所以使用tf.argmax()来得到正确答案对应的类别编号。
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.argmax(y_, 1), logits=y)
    #计算在当前batch中所有样例的交叉熵的平均值
    cross_entropy_mean = tf.reduce_mean(cross_entropy)
    #计算L2正则化的损失函数
    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
    #计算模型的正则化损失，一般只计算权重的正则化损失，而不是用偏置项
    regularization  = regularizer(weights1) + regularizer(weights2)
    #总损失等于交叉熵损失与正则化损失的和
    loss = cross_entropy_mean +regularization
    #设置指数衰减的学习率
    learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE,LEARNING_RATE_DECAY)
    #使用GD梯度下降优化算法优化损失函数，注意这里的损失函数包含了交叉熵损失和L2正则化损失
    train_step=tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_step)
    #在训练神经网络模型时，每过一遍数据急需要通过反向传播来更新神经网络的参数，又要更新每一个参数的滑动平均值。为了一次完成操作：
    #tf.control_dependencies()\tf.group()两种机制均能实现
    #train_OP = tf.group(train_step,variable_averages_op)
    with tf.control_dependencies([train_step, variables_averages_op]):
        train_op = tf.no_op(name='train')
        #检验使用了滑动均值模型的神经网络前向传播结果是否正确。tf.argmax(average_y,1)计算每一个样例的预测答案。其中average_y是一个batch_size*10的二维
    #数组，每一行表示一个样例的前向传播结果，tf.argmax的第二个参数‘1’表示选取最大值的操作在第一个维度中进行，也就是说，只在每一行选择最大值对应的下标。
    #于是得到的结果是一个长度为batch的一维数组，这个一维数组中的值就表示了每一个样例对应的数字识别的结果。
    #tf.equal()判断两个张量是否相等。
    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_,1))
    #首先讲bool值转化为数值，然后局算平均值，这个平均值就是模型在这一组数据中的正确率
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    
    #初始会话，并开始训练过程
    with tf.Session() as sess:
        #将所有的参数变量初始化
        tf.initialize_all_variables().run()
        #准备验证数据，一般在神经网络的训练过程中会通过验证数据来大致判断停止的条件和评判训练的结果
        validata_feed = {x:mnist.validation.images, y_:mnist.validation.labels}
        #准备测试数据，在真实的应用中，这部分数据在训练时是不可见的，这个数据只是作为模型优劣的最后评价标准
        test_feed = {x:mnist.test.images, y_:mnist.test.labels}
        #迭代训练神经网络
        for i in range(TRAINING_STEPS):
            #每一千轮输出一次在验证集上的测试结果
            if i % 1000 == 0:
                validata_acc = sess.run(accuracy, feed_dict=validata_feed)
                print("After %d training step(s), validation accuracy"
                      "using average model is %g" % (i, validata_acc))

            #产生这一轮使用的一个batch数据，并运行训练过程
            xs, ys = mnist.train.next_batch(BATCH_SIZE)
            sess.run(train_op, feed_dict={x:xs, y_:ys})

        #在训练结束之后，在测试集上检测神经网络模型的最终正确率
        test_acc = sess.run(accuracy, feed_dict=test_feed)
        print("After %d training step(s), test accuracy using average"
              "model is %g" % (TRAINING_STEPS, test_acc))

#主程序入口
def main(argv=None):
    #声明处理MNIST数据集的类，这个类在初始化时会调用数据集
    mnist = input_data.read_data_sets("./MNIST_data/", one_hot=True)
    train(mnist)

#TensorFlow提供一个主程序的入口，tf.app.run回调用上面定义的main函数
if __name__ == '__main__':
    tf.app.run()

构图逻辑是：先定义需要的参数，接着定义神经网络训练模型，最后调用函数进行训练。
最后附上我的输出结果仅供参考，警告没关系。

C:\Users\Administrator\Anaconda3\python.exe E:/PyCharm2007/ProgramingDM/.idea/MNIST_Tensorflow/MNIST.py
WARNING:tensorflow:From E:/PyCharm2007/ProgramingDM/.idea/MNIST_Tensorflow/MNIST.py:95: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use alternatives such as official/mnist/dataset.py from tensorflow/models.
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.
Instructions for updating:
Please write your own downloading logic.
Extracting ./MNIST_data/train-images-idx3-ubyte.gz
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.data to implement this functionality.
Extracting ./MNIST_data/train-labels-idx1-ubyte.gz
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.data to implement this functionality.
Extracting ./MNIST_data/t10k-images-idx3-ubyte.gz
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.one_hot on tensors.
Extracting ./MNIST_data/t10k-labels-idx1-ubyte.gz
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use alternatives such as official/mnist/dataset.py from tensorflow/models.
2018-12-11 15:39:59.923878: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
WARNING:tensorflow:From C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\util\tf_should_use.py:189: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.
Instructions for updating:
Use `tf.global_variables_initializer` instead.
After 0 training step(s), validation accuracyusing average model is 0.1068
After 1000 training step(s), validation accuracyusing average model is 0.9772
After 2000 training step(s), validation accuracyusing average model is 0.9828
After 3000 training step(s), validation accuracyusing average model is 0.9848
After 4000 training step(s), validation accuracyusing average model is 0.9856
After 5000 training step(s), validation accuracyusing average model is 0.9854
After 6000 training step(s), validation accuracyusing average model is 0.9848
After 7000 training step(s), validation accuracyusing average model is 0.9848
After 8000 training step(s), validation accuracyusing average model is 0.9842
After 9000 training step(s), validation accuracyusing average model is 0.9846
After 10000 training step(s), validation accuracyusing average model is 0.9854
After 11000 training step(s), validation accuracyusing average model is 0.9848
After 12000 training step(s), validation accuracyusing average model is 0.9858
After 13000 training step(s), validation accuracyusing average model is 0.9864
After 14000 training step(s), validation accuracyusing average model is 0.9864
After 15000 training step(s), validation accuracyusing average model is 0.986
After 16000 training step(s), validation accuracyusing average model is 0.9856
After 17000 training step(s), validation accuracyusing average model is 0.986
After 18000 training step(s), validation accuracyusing average model is 0.9864
After 19000 training step(s), validation accuracyusing average model is 0.9862
After 20000 training step(s), validation accuracyusing average model is 0.9858
After 21000 training step(s), validation accuracyusing average model is 0.986
After 22000 training step(s), validation accuracyusing average model is 0.9864
After 23000 training step(s), validation accuracyusing average model is 0.9862
After 24000 training step(s), validation accuracyusing average model is 0.9858
After 25000 training step(s), validation accuracyusing average model is 0.987
After 26000 training step(s), validation accuracyusing average model is 0.9858
After 27000 training step(s), validation accuracyusing average model is 0.9864
After 28000 training step(s), validation accuracyusing average model is 0.9864
After 29000 training step(s), validation accuracyusing average model is 0.9868
After 30000 training step(s), test accuracy using averagemodel is 0.9831