Tensorflow 学习笔记

摘要

研二可能搞大数据，Spark和数据挖掘、机器学习什么的可能都要用到，做个笔记，顺带分享

Reference
[1]. http://wiki.jikexueyuan.com/project/tensorflow-zh/tutorials/deep_cnn.html
[2]. http://v.youku.com/v_show/id_XMjc3MzA3OTM0NA==.html?spm=a2hzp.8253876.0.0&f=27327189

基于Win10 Tensorflow 0.12(CPU)

简单的实例

(1) 一个最简单的直线方程

import tensorflow as tf
import numpy as np

# create data
print('create data...')
x_data = np.random.rand(100).astype(np.float32)
y_data = x_data * 0.1 + 0.3


# create tensorflow structure start
Weights = tf.Variable(tf.random_uniform([1],-1.0,1.0))
biases = tf.Variable(tf.zeros([1]))

y = Weights*x_data + biases

loss = tf.reduce_mean(tf.square(y-y_data))
learn_effciency = 0.5 #学习效率一般小于1
optimizer = tf.train.GradientDescentOptimizer(learn_effciency)
train = optimizer.minimize(loss)

init = tf.initialize_all_variables() #初始化所有变量


# Start the neutral network
sess = tf.Session()
sess.run(init)

for step in range(201):
    sess.run(train)
    if step%20==0 :
        print(step,sess.run(Weights),sess.run(biases))

(2) Session的两种使用方式

import tensorflow as tf
import numpy as np

matrix1 = tf.constant([[3,3]])
matrix2 = tf.constant([[2],[2]])

product = tf.matmul(matrix1,matrix2) # matrix multiply == np.dot(m1,m2)

# # Session usage type 1
# sess = tf.Session()
# result = sess.run(product)
# print(result)
# sess.close()

# Session usage type 2
with tf.Session() as sess: # 类似于一个auto_ptr
    result2 = sess.run(product)
    print(result2)

(3) 使用变量一定要记得run初始化哦

import tensorflow as tf

state = tf.Variable(0, name='counter')
one = tf.constant(1)

new_value = tf.add(state,one)
update = tf.assign(state, new_value)

# 必须初始化所有变量

# Deprecated and will be removed after 2017-03-02.
# init = tf.initialize_all_variables() 
# with tf.Session() as sess:
#     sess.run(init)
#     for _ in range(3):
#         sess.run(update)
#         print(sess.run(state))

# Latest solution
init_op = tf.global_variables_initializer()
local_init_op = tf.local_variables_initializer()  # local variables like epoch_num, batch_size 可以不初始化local
with tf.Session() as sess:
    sess.run(init_op)
    sess.run(local_init_op)
    for _ in range(3):
        sess.run(update)
        print(sess.run(state))

(4) 使用placeholder预定义变量

import tensorflow as tf

input1 = tf.placeholder(tf.float32)
input2 = tf.placeholder(tf.float32)

output = tf.multiply(input1,input2) # tf.mul已经用不了了

with tf.Session() as sess:
    print(sess.run(output,feed_dict={input1:[7.], input2:[2.]})) # 补上数据

(5) 添加神经层

import tensorflow as tf

# 添加神经层
def add_layer(inputs, in_size, out_size, activation_function=None):
    Weights = tf.Variable(tf.random_normal([in_size, out_size]))
    biases = tf.Variable(tf.zeros([1, out_size]) + 0.1)
    Wx_plus_b = tf.matmul(inputs, Weights) + biases # 没有被激活的值

    if activation_function is None: # 说明是线性方程
        outputs = Wx_plus_b
    else:
        outputs = activation_function(Wx_plus_b)

    return outputs

(6) 一个只有一层hidden layer的简单学习

import tensorflow as tf
import numpy as np

# 添加神经层
def add_layer(inputs, in_size, out_size, activation_function=None):
    Weights = tf.Variable(tf.random_normal([in_size, out_size]))
    biases = tf.Variable(tf.zeros([1, out_size]) + 0.1)
    Wx_plus_b = tf.matmul(inputs, Weights) + biases # 没有被激活的值

    if activation_function is None: # 说明是线性方程
        outputs = Wx_plus_b
    else:
        outputs = activation_function(Wx_plus_b)

    return outputs


x_data = np.linspace(-1,1,300)[:,np.newaxis] # 后面的中括号用于加维度
noise = np.random.normal(0,0.05,x_data.shape) # 噪声，用来模拟真实数据
y_data = np.square(x_data) - 0.5 + noise


xs = tf.placeholder(tf.float32, [None, 1]) # None表示无论给多少个sample都可以
ys = tf.placeholder(tf.float32, [None, 1])


# 1 -> 10 -> 1
l1 = add_layer(xs, 1, 10, activation_function=tf.nn.relu)
prediction = add_layer(l1, 10, 1, activation_function=None)

loss = tf.reduce_mean(   # 求平均数
          tf.reduce_sum( # 对例子的差进行求和
            tf.square(ys - prediction), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)

init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)
    for i in range(1000):
        sess.run(train_step,feed_dict={xs:x_data, ys:y_data})
        if i% 50==0:
            print(sess.run(loss,feed_dict={xs:x_data, ys:y_data}))

(7) 训练过程可视化

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

# 添加神经层
def add_layer(inputs, in_size, out_size, activation_function=None):
    Weights = tf.Variable(tf.random_normal([in_size, out_size]))
    biases = tf.Variable(tf.zeros([1, out_size]) + 0.1)
    Wx_plus_b = tf.matmul(inputs, Weights) + biases # 没有被激活的值

    if activation_function is None: # 说明是线性方程
        outputs = Wx_plus_b
    else:
        outputs = activation_function(Wx_plus_b)

    return outputs


x_data = np.linspace(-1,1,300)[:,np.newaxis] # 后面的中括号用于加维度
noise = np.random.normal(0,0.05,x_data.shape)
y_data = np.square(x_data) - 0.5 + noise


xs = tf.placeholder(tf.float32, [None, 1]) # None表示无论给多少个sample都可以
ys = tf.placeholder(tf.float32, [None, 1])


# 1 -> 10 -> 1
l1 = add_layer(xs, 1, 10, activation_function=tf.nn.relu)
prediction = add_layer(l1, 10, 1, activation_function=None)

loss = tf.reduce_mean(   # 求平均数
          tf.reduce_sum( # 对10个例子的差进行求和
            tf.square(ys - prediction), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)

init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)

    fig = plt.figure()
    # 规定坐标轴上下界
    v = [-2, 2, -2, 2]
    plt.axis(v)

    ax = fig.add_subplot(1,1,1)
    ax.scatter(x_data,y_data)
    plt.ion() # python 3.5+ 防止show()阻塞屏幕
    plt.show()

    for i in range(1000):
        sess.run(train_step,feed_dict={xs:x_data, ys:y_data})
        if i% 5==0:
            # print(sess.run(loss,feed_dict={xs:x_data, ys:y_data}))
            try:
                ax.lines.remove(lines[0])
            except Exception:
                pass
            prediction_value = sess.run(prediction,feed_dict={xs:x_data})
            lines = ax.plot(x_data,prediction_value,'r-',lw=5)
            plt.pause(0.1)

(8) 过拟合以及tensorboard

// tensorboard 使用方法

// 1. cd到logs文件夹的父目录
// 2. 使用以下命令
tensorboard --logdir='logs'

// 3. 用浏览器打开指定的网页地址
Starting TensorBoard b'47' at http://0.0.0.0:6006
// 也就是http://0.0.0.0:6006

// 注意：Windows中tensorboard经常gg, 推荐使用Mac

import tensorflow as tf
from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelBinarizer

# load data
digits = load_digits()
X = digits.data
y = digits.target
y = LabelBinarizer().fit_transform(y)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3)


# 添加神经层
def add_layer(inputs, in_size, out_size, layer_name, activation_function=None):
    Weights = tf.Variable(tf.random_normal([in_size, out_size]))
    biases = tf.Variable(tf.zeros([1, out_size]) + 0.1)
    Wx_plus_b = tf.matmul(inputs, Weights) + biases # 没有被激活的值

    # Dropout 一部分的神经元，减小过拟合
    Wx_plus_b = tf.nn.dropout(Wx_plus_b, keep_prob)

    if activation_function is None: # 说明是线性方程
        outputs = Wx_plus_b
    else:
        outputs = activation_function(Wx_plus_b)
    tf.summary.histogram(layer_name+'/outputs', outputs) # ver 1.0 以后必须这么写
    return outputs

keep_prob = tf.placeholder(tf.float32) # 每次保留多少百分百的值
xs = tf.placeholder(tf.float32, [None, 64]) # None表示无论给多少个sample都可以
ys = tf.placeholder(tf.float32, [None, 10])

# add output layer
l1 = add_layer(xs, 64, 50, 'l1', activation_function=tf.nn.tanh)
prediction = add_layer(l1, 50, 10, 'l2', activation_function=tf.nn.softmax)

# the loss between prediction and real data
cross_entropy = tf.reduce_mean(-tf.reduce_sum(ys*tf.log(prediction),
                                              reduction_indices=[1])) # loss
tf.summary.scalar('loss', cross_entropy)
train_step = tf.train.GradientDescentOptimizer(0.6).minimize(cross_entropy)


with tf.Session() as sess:
    merged = tf.summary.merge_all()
    # summary writer goes in here
    train_writer = tf.summary.FileWriter("logs/train",sess.graph)
    test_writer  = tf.summary.FileWriter("logs/test",sess.graph)

    sess.run(tf.global_variables_initializer())

    for i in range(1000):
        sess.run(train_step,feed_dict={xs:X_train, ys:y_train, keep_prob: 0.5})
        if i% 50==0:
            # record loss
            train_result = sess.run(merged, feed_dict={xs:X_train, ys:y_train, keep_prob: 1}) #记录的时候不用drop任何东西
            test_result  = sess.run(merged, feed_dict={xs:X_test, ys:y_test, keep_prob: 1})
            train_writer.add_summary(train_result,i)
            test_writer.add_summary(test_result,i)

(9) 卷积神经网络

参考以下文章
http://blog.csdn.net/wang_junjie/article/details/51317348

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import  input_data

# number 1 to 10 data
mnist = input_data.read_data_sets('MNIST_data',one_hot=True)

def compute_accuracy(v_xs, v_ys):
    global prediction
    y_pre = sess.run(prediction, feed_dict={xs:v_xs, keep_prob:1})
    correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(v_ys,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    result = sess.run(accuracy, feed_dict={xs:v_xs, ys:v_ys, keep_prob:1})
    return  result

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1) # 产生随机变量
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')
    # x:图片的所有信息， W:权值， strides:卷积步长
    # strides规定为一个[1, x_movement, y_movement, 1],
    # 第一个和第四个值必须为1，第二个值为X方向步长，第三个值为Y方向的步长
    # padding : (1)SAME  : 抽取包括边缘值以外的值，以0填充，卷积后图片大小和原图片一致
    #           (2)VALID : 只抽取边缘内的值，卷积后图片大小比原图片小

def max_pool_2x2(x): # Pooling技术保留更多的图片信息
    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
    # ksize第一个和第四个值必须为1


# define placeholder for inputs to nerwork
xs = tf.placeholder(tf.float32,[None,784])
ys = tf.placeholder(tf.float32,[None,10])
keep_prob = tf.placeholder(tf.float32)
x_image = tf.reshape(xs,[-1,28,28,1]) # -1:表示不管这个维度, 1:图片高度

## conv1 layer ##
W_conv1 = weight_variable([5,5,1,32]) # patch 5x5 卷积器， in_size 1 图片的输入高度, output_size 32
b_conv1 = bias_variable([32])
h_conv1 = tf.nn.relu(conv2d(x_image,W_conv1) + b_conv1) # relu:非线性化, output_size 28x28x32 采用SAME所以长宽不变 而卷积输出高度为32
h_pool1 = max_pool_2x2(h_conv1) # output_size 14x14x32

## conv2 layer ##
W_conv2 = weight_variable([5,5,32,64]) # patch 5x5 卷积器， in_size 32 图片的输入高度, output_size 64
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1,W_conv2) + b_conv2) # relu:非线性化, output_size 14x14x64 采用SAME所以长宽不变 而卷积输出高度为32
h_pool2 = max_pool_2x2(h_conv2) # output_size 7x7x64

## func1 layer ##
W_fc1 = weight_variable([7*7*64,1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2,[-1,7*7*64]) # 数据一维化 [n_samples,7,7,64] -> [n_samples,7*7*64]
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1)+b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

## func2 layer ##
W_fc2 = weight_variable([1024,10])
b_fc2 = bias_variable([10])
prediction = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2)+b_fc2) # softmax classification算概率

# the error between prediction and real data
cross_entropy = tf.reduce_mean(-tf.reduce_sum(ys*tf.log(prediction), reduction_indices=[1]))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for i in range(1000):
        batch_xs, batch_ys = mnist.train.next_batch(100)
        sess.run(train_step, feed_dict={xs:batch_xs, ys:batch_ys, keep_prob:0.5})
        if i % 50 == 0:
            print(compute_accuracy(mnist.test.images, mnist.test.labels))

(10) 保存读取Weights和Bias

import tensorflow as tf
import numpy as np


## Save to file
W = tf.Variable([[1,2,3],[3,4,5]], dtype=tf.float32, name='weights')
b = tf.Variable([[1,2,3]], dtype=tf.float32, name='bias')

saver = tf.train.Saver()

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    # 一定要先建立一个my_net文件夹
    save_path = saver.save(sess, "my_net/save_net.ckpt")
    print("Save to path:", save_path)


## Restore variables
# restore时一定要定义dtype 和 shape
W = tf.Variable(np.arange(6).reshape((2,3)), dtype=tf.float32, name='weights')
b = tf.Variable(np.arange(3).reshape((1,3)), dtype=tf.float32, name='bias')

# not need init step
saver = tf.train.Saver()
with tf.Session() as sess:
    saver.restore(sess,"my_net/save_net.ckpt")
    print("weights:",sess.run(W))
    print("bias:",sess.run(b))

(11) RNN

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# number 1 to 10 data
mnist = input_data.read_data_sets('MNIST_data',one_hot=True)


#hyper parameters
lr = 0.001
training_iters = 100000
batch_size = 128
display_step = 10

n_inputs = 28 # MNIST data input (img shape:28x28)
n_steps  = 28 # time steps
n_hidden_units = 128 # neurons in hidden layer
n_classes = 10 # MNIST classes (0-9) 分为10个类

# tf Graph input
x = tf.placeholder(tf.float32,[None, n_steps, n_inputs])
y = tf.placeholder(tf.float32,[None, n_classes])

# Define weights
weights = {
    #(28,128)
    'in':tf.Variable(tf.random_normal([n_inputs, n_hidden_units])),
    #(128,10)
    'out':tf.Variable(tf.random_normal([n_hidden_units, n_classes]))
}
biases = {
    #(128,)
    'in':tf.Variable(tf.constant(0.1, shape=[n_hidden_units,])),
    #(10,)
    'out':tf.Variable(tf.constant(0.1, shape=[n_classes,]))
}

def RNN(X, weights, biases):
    # hidden layer for input to cell
    # X(128batch, 28 steps, 28 inputs)
    # ==> (128*28, 28 inputs)
    X = tf.reshape(X,[-1,n_inputs])
    # ==>(128batch * 28steps, 128 hidden)
    X_in = tf.matmul(X, weights['in']) + biases['in']
    # ==>(128batch , 28steps, 128 hidden)
    X_in= tf.reshape(X_in,[-1,n_steps,n_hidden_units])

    # cell
    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden_units, forget_bias=1.0, state_is_tuple=True)
    _init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32)

    outputs, states = tf.nn.dynamic_rnn(lstm_cell, X_in, initial_state=_init_state, time_major=False)

    # hidden layer for output as the final results
    results = tf.matmul(states[1], weights['out']) + biases['out']

    return results


pred = RNN(x, weights, biases)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
train_op = tf.train.AdamOptimizer(lr).minimize(cost)

correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    step = 0
    while step*batch_size < training_iters:
        batch_xs, batch_ys = mnist.train.next_batch(batch_size)
        batch_xs = batch_xs.reshape([batch_size, n_steps, n_inputs])
        sess.run([train_op],feed_dict={
            x: batch_xs,
            y: batch_ys
        })
        if step % 100 == 0:
            print(sess.run(accuracy,feed_dict={
                x: batch_xs,
                y: batch_ys
            }))