卷积神经网络结构及tensorflow实现（CNN，ResNet，DenseNet）

        本文介绍三种卷积神经网络结构，分别是CNN、ResNet、DenseNet，以及利用python和tensorflow分别实现这三种神经网络结构，在MNIST手写数字集实现图像分类，

一. CNN（卷积神经网络）

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)   #import MNIST
print("start")

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')#train data
y = tf.placeholder(tf.float32, [None, 10])#train label
sess = tf.InteractiveSession()

def weight_variable(shape):
#initial variable
  initial = tf.truncated_normal(shape, mean=0,stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

x = tf.reshape(inputs_, (-1,28,28,1))   #x为-1*28*28*1

conv1 = tf.layers.conv2d(x, 32, (3,3), padding='same', activation=tf.nn.relu)
pooling1 = tf.layers.max_pooling2d(conv1, (2,2), (2,2), padding='same')
#conv1 turn to 14*14*32
conv2 = tf.layers.conv2d(pooling1, 64, (3,3), padding='same', activation=tf.nn.relu)
pooling2 = tf.layers.max_pooling2d(conv2, (2,2), (2,2), padding='same')
#7*7*64
conv3 = tf.layers.conv2d(pooling2, 64, (3,3), padding='same', activation=tf.nn.relu)
pooling3 = tf.layers.max_pooling2d(conv3, (2,2), (2,2), padding='same')
#4*4*64
#fully connected network
flat = tf.reshape(pooling3, [-1,4*4*64])  #turn to 1 dim

w_fc1 = weight_variable([4 * 4 *64, 1024])
b_fc1 = bias_variable([1024])

h_fc1 = tf.nn.relu(tf.matmul(flat, w_fc1) + b_fc1)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

prediction = tf.nn.softmax(y_conv,name='prediction')    #softmax返回一组概率向量
#建立损失函数，在这里采用交叉熵函数
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))
#最小化损失
train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1))  #返回bool型(结果，标签)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))  #求平均，计算正确率（tf.cast:bool-float转换）
#初始化变量

#sess = tf.Session()
sess.run(tf.global_variables_initializer())
pos = 0
trains = []
print("houyu")
for i in range(6001):
    batch = mnist.train.next_batch(100)
    images = batch[0].reshape((-1,28,28,1)) #像素归一化为[0,1]之间的值
    #print("images shape:",np.shape(images))
    labels = batch[1]
    if i%100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:images, y: labels, keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: images, y: labels, keep_prob: 0.5})

二. ResNet（残差神经网络）

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')
y = tf.placeholder(tf.float32, [None, 10])
sess = tf.InteractiveSession()

def weight_variable(shape):
#这里是构建初始变量
  initial = tf.truncated_normal(shape, mean=0,stddev=0.1) #s生成正态分布的随机数
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

#定义残差网络的identity_block块(输入和输出维度相同)
def identity_block(X_input, kernel_size, in_filter, out_filters, stage, block):
        """
        Arguments:
        X_input -- input tensor of shape (m, H, W, C)
        kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
        filter -- python list of integers, defining the number of filters in the CONV layers of the main path
        stage -- integer, used to name the layers, depending on their position in the network
        block -- string/character, used to name the layers, depending on their position in the network

        Returns:
        X -- output of the identity block, tensor of shape (n, H, W, C)
        """

        # defining name basis
        block_name = 'res' + str(stage) + block
        f1, f2, f3 = out_filters
        with tf.variable_scope(block_name):
            X_shortcut = X_input

            #first
            X = tf.layers.conv2d(X_input, f1, (1,1), padding='SAME', activation=tf.nn.relu)

            #second
            X = tf.layers.conv2d(X, f2, (kernel_size, kernel_size), padding='SAME', activation=tf.nn.relu)

            #third
            X = tf.layers.conv2d(X, f3, (1,1), padding='SAME', activation=tf.nn.relu)
            #final step
            add = tf.add(X, X_shortcut)
            b_conv_fin = bias_variable([f3])
            add_result = tf.nn.relu(add+b_conv_fin)

        return add_result

#定义conv_block模块，由于该模块定义时输入和输出尺度不同，需要进行卷积操作来改变尺度，从而相加
def convolutional_block( X_input, kernel_size, in_filter,
                            out_filters, stage, block, stride=1):
        """
        Arguments:
        X -- input tensor of shape (m, H, W, C)
        kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
        filters -- python list of integers, defining the number of filters in the CONV layers of the main path
        stage -- integer, used to name the layers, depending on their position in the network
        block -- string/character, used to name the layers, depending on their position in the network
        stride -- Integer, specifying the stride to be used

        Returns:
        X -- output of the convolutional block, tensor of shape (n, H, W, C)
        """

        # defining name basis
        block_name = 'res' + str(stage) + block
        with tf.variable_scope(block_name):
            f1, f2, f3 = out_filters

            x_shortcut = X_input
            #filter 1*1*f1
            X = tf.layers.conv2d(X_input, f1, (1,1), padding='SAME', activation=tf.nn.relu)

            #second 3*3*f2
            X = tf.layers.conv2d(X, f2, (kernel_size, kernel_size), padding='SAME', activation=tf.nn.relu)

            #third 1*1*f3
            X = tf.layers.conv2d(X, f3, (1, 1), padding='SAME', activation=tf.nn.relu)
            #shortcut path
            W_shortcut =weight_variable([1, 1, in_filter, f3])
            x_shortcut = tf.nn.conv2d(x_shortcut, W_shortcut, strides=[1, stride, stride, 1], padding='VALID')

            #final
            add = tf.add(x_shortcut, X)
            #建立最后融合的权重
            b_conv_fin = bias_variable([f3])
            add_result = tf.nn.relu(add + b_conv_fin)

        return add_result

x = tf.reshape(inputs_, (-1,28,28,1))

x1 = convolutional_block(x, 3, 1, [32, 32, 64], stage=1, block='a' )
pooling1 = tf.layers.max_pooling2d(x1, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#这里操作后变成14x14x64

x2 = convolutional_block(X_input=pooling1, kernel_size=3, in_filter=64,  out_filters=[64, 64, 128], stage=2, block='b', stride=1)
pooling2 = tf.layers.max_pooling2d(x2, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#上述conv_block操作后，尺寸变为7x7x128

x3 = identity_block(pooling2, 3, 128, [64, 64, 128], stage=2, block='c' )
pooling3 = tf.layers.max_pooling2d(x3, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#上述操作后张量尺寸变成4*4*128

flat = tf.reshape(pooling3, [-1,4*4*128])   #将向量拉直

w_fc1 = weight_variable([4 * 4 * 128, 1024])
b_fc1 = bias_variable([1024])
h_fc1 = tf.nn.relu(tf.matmul(flat, w_fc1) + b_fc1)

keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

#建立损失函数，在这里采用交叉熵函数
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))

train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #cast将bool型转为float，求均值得到正确率
#初始化变量

sess.run(tf.global_variables_initializer())

print("start")
for i in range(2000):
    batch = mnist.train.next_batch(100)
    input = batch[0].reshape((-1,28,28,1))
    if i % 100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:input, y: batch[1], keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: input, y: batch[1], keep_prob: 0.5})

三. DenseNet（稠密神经网络）

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.01)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.01, shape=shape)
    return tf.Variable(initial)

def conv2d(input, in_features, out_features, kernel_size, with_bias=False):
    W = weight_variable([ kernel_size, kernel_size, in_features, out_features ])
    conv = tf.nn.conv2d(input, W, [ 1, 1, 1, 1 ], padding='SAME')
    if with_bias:
        return conv + bias_variable([ out_features ])
    return conv

def avg_pool(input, s):
    return tf.nn.avg_pool(input, [ 1, s, s, 1 ], [1, s, s, 1 ], 'SAME')

def batch_activ_conv(current, in_features, out_features, kernel_size, is_training, keep_prob):
    current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
    current = tf.nn.relu(current)
    current = conv2d(current, in_features, out_features, kernel_size)
    current = tf.nn.dropout(current, keep_prob)
    return current
#定义稠密神经网络的稠密块
def block(input, layers, in_features, growth, is_training, keep_prob):
    current = input
    features = in_features
    for idx in range(layers):
        tmp = batch_activ_conv(current, features, growth, 3, is_training, keep_prob)
        current = tf.concat((current, tmp), axis=3)
        features += growth
    return current, features

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')
y = tf.placeholder(tf.float32, [None, 10])

keep_prob = tf.placeholder(tf.float32)
#is_training = tf.placeholder("bool", shape=[])
is_training = True
sess = tf.InteractiveSession()

x = tf.reshape(inputs_, (-1,28,28,1))
conv1 = conv2d(x, 1, 16, 3)
#这里操作后变成28*28*16
current, features = block(conv1, layers=4, in_features = 16, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training = is_training, keep_prob = keep_prob)
current = avg_pool(current, 2)

current, features = block(current, layers = 4, in_features = features, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training = is_training, keep_prob = keep_prob)
current = avg_pool(current, 2)

current, features = block(current, layers = 4, in_features = features, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
current = tf.nn.relu(current)
# current = avg_pool(current, 8)
current = avg_pool(current, 2)
final_dim = features
label_count = 1024
flat = tf.reshape(current, [-1, 4*4*final_dim])
Wfc = weight_variable([4*4*final_dim, label_count])
bfc = bias_variable([label_count])

h_fc1 = tf.nn.relu(tf.matmul(flat, Wfc) + bfc)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])

ys_ = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

#建立损失函数，在这里采用交叉熵函数
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=ys_))

train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(ys_,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#初始化变量

sess.run(tf.global_variables_initializer())

print("start")
for i in range(2000):
    batch = mnist.train.next_batch(100)
    input = batch[0].reshape((-1,28,28,1))
    if i % 100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:input, y: batch[1], keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: input, y: batch[1], keep_prob: 1.0})

残差神经网络：