实现代码：
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
INPUT_NODE = 784#输入结点数
OUTPUT_NODE = 10#输出结点数
LAYER1_NODE = 500#隐藏层结点数
BATCH_SIZE = 100#每批训练多少样本
LEARNING_RATE_BASE = 0.8#基础学习速率
LEARNING_RATE_DECAY = 0.99#学习率的衰减率
REGULARIZATION_RATE = 0.0001#正则化项系数
TRAINING_STEPS = 30000#训练轮数
MOVING_AVERAGE_DECAY = 0.99#滑动平均衰减率
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
#如果不采用滑动平均模型
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
#如果采用滑动平均模型
else:
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name="x-input")
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name="y-input")
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))
#未采取滑动平均模型的前向传播
y = inference(x, None, weights1, biases1, weights2, biases2)
global_step = tf.Variable(0, trainable=False)
#创建滑动平均模型
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variable_averages_op = variable_averages.apply(tf.trainable_variables())
#采取滑动平均模型的前向传播
average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)
#自带softmax处理的交叉熵误差集，sparse_softmax_cross_entropy_with_logits对只有一个正确答案的分类进行的加速处理
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_,1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)
#正则化项
regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
regularization = regularizer(weights1) + regularizer(weights2)
#最终的损失函数
loss = cross_entropy_mean + regularization
#采取指数衰减的学习速率
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
"""
反向传播时，先更新参数，再滑动平均。tf.control_dependencies绑定了这两种操作
等同于：train_op = tf.group(train_step, variable_averages_op)
"""
with tf.control_dependencies([train_step, variable_averages_op]):
train_op = tf.no_op(name='train')
#判断两个张量的每一维是否相等，相等返回True
corrent_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(corrent_prediction, tf.float32))
with tf.Session() as sess:
tf.global_variables_initializer().run()
#验证集
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
#测试集
test_feed = {x: mnist.test.images, y_: mnist.test.labels}
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g"
% (i, validate_acc))
xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})
test_acc = sess.run(accuracy, feed_dict=test_feed)
print("After %d training step(s), test accuracy using average model is %g"
% (i, test_acc))
def main(argv=None):
mnist = mnist = input_data.read_data_sets('./MNIST_data/', one_hot=True)
train(mnist)
if __name__ == '__main__':
tf.app.run()
使用验证数据集判断模型效果
为了测评神经网络模型在不同参数下的效果，一般会从训练数据中抽取一部分作为验证数据。使用验证数据就可以评判不同参数取值下模型的表现。除了使用验证数据集，还可以使用交叉验证的方式来验证模型效果。但因为神经网络训练时间本身就比较长，采用交叉验证会花费大量时间。所以在海量数据的情况下，一般会更多的采用验证数据集的形式来测评模型的效果。
变量管理
除了tf.Variable()，还有另一种创建变量的方式。下面是两种方式创建同一个变量的样例：
v = tf.get_variable("v", shape=[1], initializer=tf.constant_initializer(1.0))
v = tf.Variable(tf.constant(1.0 ,shape=[1]), name="v")
几种变量的初始化函数：
tf.constant_initializer()#常量
tf.random_normal_initializer()#正态分布的随机值
tf.truncated_normal_initializer()#正态分布偏离均值2个标准差，这个数将被重新随机
tf.random_uniform_initializer()#均匀分布的随机值
tf.uniform_unit_scaling_initializer()#初始化满足均匀分布但不影响输出数量级的随机值
tf.zeros_initializer()#变量设置全为0
tf.onses_initializer()#变量设置全为1
tf.get_variable不同于tf.Variable函数，变量名称是一个必填的参数。tf.get_variable会根据这个名字去创建或者获取变量。
#在foo的空间内创建名字为v的变量
with tf.variable_scope("foo"):
v = tf.get_variable("v", [1], initializer=tf.constant_initializer(1.0))
#因为该空间内已经有该变量，所以程序出错
with tf.variable_scope("foo"):
v = tf.get_variable("v", [1])
#如果要要获取变量，则将reuse设置为true，此时则不能获取未创建的变量
with tf.variable_scope("foo", reuse=True)
v1 = tf.get_variable("v", [1])
print v == v1
#bar中尚未创建变量v，所以程序出错
with tf.variable_scope("bar", reuse=True)
v = tf.get_variable("v", [1])
tf.variable_scope()是可以多层嵌套的，当未指定reuse的值的时候，该属性与外层的一致。
通过这函数，就可以更好的管理我们的变量了。于是我们对我们代码的inference部分进行改进：
def inference(input_tensor, reuse=False):
with tf.variable_scope("layer1", reuse=reuse):
weights = tf.get_variable("weights", [INPUT_NODE, LAYER1_NODE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
biases = tf.get_variable("biases", [LAYER1_NODE],
initializer=tf.constant_initializer(0.0))
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights) + biases)
with tf.variable_scope("layer2", reuse=reuse):
weights = tf.get_variable("weights", [LAYER1_NODE, OUTPUT_NODE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
biases = tf.get_variable("biases", [OUTPUT_NODE],
initializer=tf.constant_initializer(0.0))
layer2 = tf.matmul(layer1, weights) + biases
return layer2
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name="x-input")
y = inference(x)
TensorFlow模型持久化
持久化代码的实现：
import tensorflow as tf
v1 = tf.Variable(tf.constant(1.0, shape=[1]), name="v1")
v2 = tf.Variable(tf.constant(2.0, shape=[1]), name="v2")
result = v1 + v2
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
saver.save(sess, "./persist.ckpt")
以上代码虽然只指定了一个路径，但是最终会产生三个文件，因为TensorFlow会将计算图的结构和图上参数取值分开保存。
persist.ckpt.meta保存了计算图的结构，即神经网络的结构。
persist.ckpt保存了每一个变量的取值。
persist.ckpt.index保存了一个目录下所有模型文件列表？？？
加载模型的代码：
import tensorflow as tf
v1 = tf.Variable(tf.constant(1.0, shape=[1]), name="v1")
v2 = tf.Variable(tf.constant(2.0, shape=[1]), name="v2")
result = v1 + v2
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, "./persist.ckpt")
print(sess.run(result))
#3.
如果不希望重复定义图上的运算，也可以直接加载已经持久化的图：
import tensorflow as tf
saver = tf.train.import_meta_graph("./persist.ckpt.meta")
with tf.Session() as sess:
saver.restore(sess,"./persist.ckpt")
print(sess.run(tf.get_default_graph().get_tensor_by_name("add:0")))
上面这个方法默认保存和加载了全部的变量，如果只需要加载部分变量，则可以在声明saver时，提供一个列表来指定需要保存或加载的变量：
saver = tf.train.Saver([v1])
除了可以选取需要加载的变量，tf.train.Saver类还支持在保存或者加载时给变量重命名。
v1 = tf.Variable(tf.constant(1.0, shape=[1]), name="other-v1")
v2 = tf.Variable(tf.constant(2.0, shape=[1]), name="other-v2")
#此时如果全部载入，就会出错，因为不存在other-v1，other-v2的变量
#这时候就需要通过一个字典，给变量重命名
saver = tf.train.Saver({"v1": v1, "v2": v2})
这样做的主要目的之一是方便使用变量的滑动平均值：
import tensorflow as tf
v = tf.Variable(0, dtype=tf.float32, name="v")
for variables in tf.global_variables():
print(variables.name)
ema = tf.train.ExponentialMovingAverage(0.99)
maintain_averages_op = ema.apply(tf.global_variables())
for variables in tf.global_variables():
print(variables.name)
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
sess.run(tf.assign(v, 10))
sess.run(maintain_averages_op)
saver.save(sess, "./persist.ckpt")
print(sess.run([v, ema.average(v)]))
当读取滑动平均值时，可以使用变量重命名：
saver = tf.train.Saver({"v/ExponentialMovingAverage": v})
with tf.Session() as sess:
saver.restore(sess, "./persist.ckpt")
print(sess.run(v))
tensorflow还提提供了一个函数，方便给滑动均值重命名：
import tensorflow as tf
v = tf.Variable(0, dtype=tf.float32, name="v")
ema = tf.train.ExponentialMovingAverage(0.99)
print(ema.variables_to_restore())
saver = tf.train.Saver(ema.variables_to_restore())
with tf.Session() as sess:
saver.restore(sess, "./persist.ckpt")
print(sess.run(v))
当我们只需要存储变量而不需要存储其他结构和信息时，我们可以通过下面方式，将变量只存放在一个文件中：
import tensorflow as tf
from tensorflow.python.framework import graph_util
v1 = tf.Variable(tf.constant(1.0, shape=[1]), name="v1")
v2 = tf.Variable(tf.constant(2.0, shape=[1]), name="v2")
result = v1 + v2
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
graph_def = tf.get_default_graph().as_graph_def()
#需要保存的计算,注意这里只给出计算的节点，而不需要加:0表示第一个输出
output_graph_def = graph_util.convert_variables_to_constants(sess, graph_def, ['add'])
#将导出的模型存入文件
with tf.gfile.GFile("./persist.pb", "wb") as f:
f.write(output_graph_def.SerializeToString())
从该文件中再读取这个变量：
import tensorflow as tf
from tensorflow.python.platform import gfile
with tf.Session() as sess:
model_filename = "./persist.pb"
with gfile.FastGFile(model_filename, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
result = tf.import_graph_def(graph_def, return_elements=["add:0"])
print(sess.run(result))
持久化及变量管理改进后代码
mnist_inference.py
import tensorflow as tf
INPUT_NODE = 784
OUTPUT_NODE = 10
LAYER1_NODE = 500
def get_weight_variable(shape, regularizer):
weights = tf.get_variable("weights", shape, initializer=tf.truncated_normal_initializer(stddev=0.1))
if regularizer != None:
tf.add_to_collection("losses", regularizer(weights))
return weights
def inference(input_tensor, regularizer):
with tf.variable_scope("layer1"):
weights = get_weight_variable([INPUT_NODE, LAYER1_NODE], regularizer)
biases = tf.get_variable("biases", [LAYER1_NODE], initializer=tf.constant_initializer(0.0))
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights) + biases)
with tf.variable_scope("layer2"):
weights = get_weight_variable([LAYER1_NODE, OUTPUT_NODE], regularizer)
biases = tf.get_variable("biases", [OUTPUT_NODE], initializer=tf.constant_initializer(0.0))
layer2 = tf.matmul(layer1,weights) + biases
return layer2
mnist_train.py
import os
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import mnist_inference
BATCH_SIZE = 100#每100条数据更新一次参数
LEARNING_RATE_BASE = 0.8#初始学习速率
LEARNING_RATE_DECAY = 0.99#学习速率衰减速度
REGULARAZTION_RATE = 0.0001#正则化系数
TRAINING_STEPS = 30000#训练的轮数
MOVING_AVERAGE_DECAY = 0.99#滑动平均模型的滑动速率
MODEL_SAVE_PATH = "./"
MODEL_NAME = "model.ckpt"
def train(mnist):
#给输入、输出占位
x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name="x-input")
y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name="y-input")
#创建l2正则化，用该正则化创建前向的推导
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
y = mnist_inference.inference(x, regularizer)
#创建一个随参数更新次数自增的变量
global_step = tf.Variable(0, trainable=False)
#创建滑动平均模型，并将其应用于所有要训练的变量
variable_average = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variable_average_op = variable_average.apply(tf.trainable_variables())
#计算带正则化的交叉熵损失
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)
loss = cross_entropy_mean + tf.add_n(tf.get_collection("losses"))
#设置学习速率
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY)
#设置优化算法
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#将滑动平均模型和训练一起绑定
with tf.control_dependencies([train_step, variable_average_op]):
train_op = tf.no_op(name="train")
saver = tf.train.Saver()
with tf.Session() as sess:
#初始化变量
tf.global_variables_initializer().run()
for i in range(TRAINING_STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
_, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
if i % 1000 ==0:
print("After %d training step(s), loss on training "
"batch is %g." % (step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def main(argv=None):
mnist = input_data.read_data_sets("./MNIST_data/", one_hot=True)
train(mnist)
if __name__ == '__main__':
tf.app.run()
mnist_eval.py
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import mnist_inference
import mnist_train
EVAL_INTERVAL_SECS = 10
def evaluate(mnist):
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name="x-input")
y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name="y-input")
#设置验证数据集
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
#不带正则项的前向计算
y = mnist_inference.inference(x, None)
#准确率计算
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#读取训练时的滑动平均值
variable_averages = tf.train.ExponentialMovingAverage(mnist_train.MOVING_AVERAGE_DECAY)
variable_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variable_to_restore)
#每隔EVAL_INTERVAL_SECS时间就计算一次正确率
while True:
with tf.Session() as sess:
#该函数会通过checkpoint文件自动找到这个目录下最新模型的文件名
ckpt = tf.train.get_checkpoint_state(mnist_train.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
accuracy_score = sess.run(accuracy, feed_dict=validate_feed)
print("After %s training step(s), validation"
"accuracy = %g" % (global_step, accuracy_score))
else:
print("No checkpoint file found")
return
time.sleep(EVAL_INTERVAL_SECS)
def main(argv=None):
mnist = input_data.read_data_sets("./MNIST_data/", one_hot=True)
evaluate(mnist)
if __name__ == '__main__':
tf.app.run()