Tensorflow学习笔记九——经典卷积神经网络

简介LeNet-5，AlexNet，VGGNet，InceptinNet-v3，ResNet
9.1 LeNet-5卷积网络模型
1.模型结构
第一层：输入层，3232分辨率黑白图像。但是mnist数据集中图片分辨率为2828.这样做的目的是希望高层调整检测感受野的中心能够手机更多潜在的明显特征，如角点，断点等。
第二层：卷积层，6@28,28，卷积核尺寸为5,5,6，因此参数数量为28x28（5x5x1x6+6）=156个。
第三层：下采样层，经过2x2最大池化，步长为2操作得到6@14x14特征图。
第四层：卷积层，卷积核大小为1@5*5。第三层和第四层之间的链接不是一对一连接，而是以固定关系连接。
第五层：下采样层，16x5x5
第六层：卷积层–全连接层
第七层：由sigmod激活函数传递到输出层
第八层：全连接层：计算径向基函数：yi=sigma(x-wi,j)2 RBF计算和第i个数字的比特图编码有关。
2.TensorFlow代码实现

##encode decode
import tensorflow as tf
from tensorflow.keras import layers
#datasets
(train_images,train_labels),(test_iamges,test_labels)=tf.keras.datasets.mnist.load_data()

train_images=train_images.astype('float32')/255
test_images=test_iamges.astype('float32')/255

train_images=train_images[...,tf.newaxis]
test_iamges=test_iamges[...,tf.newaxis]

train_dataset=tf.data.Dataset.from_sensor_slices((train_images,train_labels))
train_dataset=train_dataset.shuffle(buffer_size=1024).batch(100)

test_dataset=tf.data.Dataset.from_sensor_slices((test_images,test_labels))
test_dataset=test_dataset.batch(100)
#model
class LeNetMode(tf.keras.Model):
    def __init__(self):
        super(LeNetMode,self).__init__()

        self.conv1=layers.Conv2D(filters=32,kernal_size=(5,5),padding="SAME",activation='relu',use_bias=True,bias_initializer='zeros')
        self.maxpool1=layers.MaxPool2D(pool_size=(2,2),strides=(2,2),padding='same')
        self.conv2=layers.Conv2D(filters=64,kernal_size=(5,5),strides=(1,1),padding='SAME',activation='relu',use_bias=True,bias_initializer='zeros')
        self.maxpool2=layers.MaxPool2D(pool_size=(2,2),strides=(2,2),padding='same')
        self.flatten=layers.Flatten()

        self.full1=layers.Dense(uints=512,activation='relu',use_bias=True,bias_initializer='zeros')
        self.full2=layers.Dense(uints=10,activation='softmax',use_bias=True,bias_initializer='zeros')

    def call(self,inputs):
        x=self.conv1(inputs)
        x=self.maxpool1(x)
        x=self.conv2(x)
        x=self.maxpool2(x)
        x=self.flatten(x)
        x=self.full1(x)
        x=self.full2(x)
        return x
    model=LeNetMode()

#loss and optimizers
loss_fn=tf.keras.losses.SparseCategoricalCrossentropy()
optimzer=tf.keras.optimizers.Adam()
train_loss=tf.keras.metrics.Mean(name='train_loss')
train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')

@tf.function
def train_step(images,labels):
    with tf.GradientTape() as tape:
        predictions=model(images)
        loss=loss_fn(labels,predictions)
        gradients=tape.gradient(loss,model.trainable_variables)
        optimzer.apply_gradients(zip(gradients,model.trainable_variables))
        train_loss(loss)
        train_accuracy(labels,predictions)
        
@tf.function
def test_step(iamges,labels):
    predictions=model(images)
    t_loss=loss_fn(labels,predictions)
    
    test_loss(t_loss)
    test_accuracy(labels,predictions)
    
#start model anf train
EPOCHS=40
for epoch in range(EPOCHS):
    for images,labels in train_dataset:
        train_step(images,labels)
    
    for test_images,test_labels in test_dataset:
        test_step(test_images,test_labels)
        
    template='Epoch {},Loss {},Accuarcy:{}. Test Loss: {},Test Accuracy: {}'
    print(template.format(epoch+1,train_loss.result(),train_accuracy.result()*100,test_loss.result(),test_accuracy.result()*100))
    

LeNet-5网络无法训练较大的图像数据集。
9.2 AlexNet卷积网络模型
1.模型结构
第一段卷积(conv1):

• 输入的224,224,3图像
• 96@11，11卷积核操作，stride=(4,4)，得到96，55，55的特征图。
• ReLu去线性化
• LRN（AlexNet首次提出）
• 3*3最大池化，步长为2
• 输出96@27，27的特征图

第二段卷积(conv2):

• 输入conv1数据
• 256@5，5，3卷积核，stride=(1,1)，得到256，27，27的特征图。
• ReLu去线性化
• LRN（AlexNet首次提出）
• 3*3最大池化，步长为2
• 输出256@13，13的特征图

第三段卷积(conv3):

• 输入conv2数据
• 384@3，3卷积核操作，stride=(1,1)，得到384，13，13的特征图。
• ReLu去线性化
• 输出384@13，13的特征图

第四段卷积(conv4):

• 输入conv3数据
• 256@3，3卷积核操作，stride=(1,1)，得到256，13，13的特征图。
• ReLu去线性化
• 输出256@13，13的特征图

第五段卷积(conv5):

• 输入的conv4数据
• 256@3，3卷积核操作，stride=(1,1)，得到256，13，13的特征图。
• ReLu去线性化
• 3*3最大池化，步长为2
• 输出256@6，6的特征图

2.TensorFlow代码实现

##encode decode
import tensorflow as tf
import math
import time
from datetime import datetime

batch_size=32
num_batches=100
#datasets
image_size=224
image_shape=[batch_size,image_size,image_size,3]
image_init=tf.random_normal_initializer(stddec=1e-1)
image_data=tf.Variale(initial_value=image_init(shape=image_shape),dtype='float32')

#model
class Conv1(layers.Layer):
    def __init__(self):
        super(Conv1,self).__init__()
    def build(self,input_shape):
        self.kernel=self.add_weight(name='Conv1/kernel',
        shape=[11,11,3,96],initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name='Conv1/biases',shape=[96],initializer=b_init,
        dtype='float32',trainable=True)
    def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,4,4,1],padding="SAME")
        relu=tf.nn.relu(tf.nn.biases_add(conv,self.biases))
        lrn=tf.nn.lrn(relu,4,bias=1.0,alpha=0.001/9.0,beta=0.75,name="Conv1/lrn")
        pool=tf.nn.max_pool(lrn,ksize=[1,3,3,1],strides=[1,2,2,1],padding="VALID",name="Conv1/pool")
        return pool

class Conv2(layers.Layer):
    def __init__(self):
        super(Conv2,self).__init__()
    def build(self,input_shape):
        self.kernel=self.add_weight(name='Conv2/kernel',
        shape=[5,5,96,256],initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name='Conv2/biases',shape=[256],initializer=b_init,
        dtype='float32',trainable=True)
    def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,1,1,1],padding="SAME")
        relu=tf.nn.relu(tf.nn.biases_add(conv,self.biases))
        lrn=tf.nn.lrn(relu,4,bias=1.0,alpha=0.001/9.0,beta=0.75,name="Conv2/lrn")
        pool=tf.nn.max_pool(lrn,ksize=[1,3,3,1],strides=[1,2,2,1],padding="VALID",name="Conv1/pool")
        return pool


class Conv3(layers.Layer):
    def __init__(self):
        super(Conv3,self).__init__()
    def build(self,input_shape):
        self.kernel=self.add_weight(name='Conv3/kernel',
        shape=[3,3,256,384],initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name='Conv3/biases',shape=[384],initializer=b_init,
        dtype='float32',trainable=True)
    def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,1,1,1],padding="SAME")
        relu=tf.nn.relu(tf.nn.biases_add(conv,self.biases))
        return relu


class Conv4(layers.Layer):
    def __init__(self):
        super(Conv4,self).__init__()
    def build(self,input_shape):
        self.kernel=self.add_weight(name='Conv4/kernel',
        shape=[3,3,384,384],initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name='Conv4/biases',shape=[384],initializer=b_init,
        dtype='float32',trainable=True)
    def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,1,1,1],padding="SAME")
        relu=tf.nn.relu(tf.nn.biases_add(conv,self.biases))
        return relu

class Conv5(layers.Layer):
    def __init__(self):
        super(Conv4,self).__init__()
    def build(self,input_shape):
        self.kernel=self.add_weight(name='Conv4/kernel',
        shape=[3,3,384,384],initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name='Conv4/biases',shape=[384],initializer=b_init,
        dtype='float32',trainable=True)
    def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,1,1,1],padding="SAME")
        relu=tf.nn.relu(tf.nn.biases_add(conv,self.biases))
        return relu

class AlexNet(tf.keras.Model):
    def __init__(self):
        super(AlexNet,self).__init__()
        self.conv1=Conv1()
        self.conv2=Conv2()
        self.conv3=Conv3()
        self.conv4=Conv4()
        self.conv5=Conv5()
        self.flatten=layers.Flatten()
        self.dense1=layers.Dense(uints=4096,activation='relu')
        self.dense2=layers.Dense(uints=4096,activation='relu')
    def call(self,input):
        x=self.conv1(x)
        x=self.conv2(x)
        x=self.conv3(x)
        x=self.conv4(x)
        x=self.conv5(x)
        x=self.flatten(x)
        x=self.dense1(x)
        return self.dense2(x)

#print net message
alexnet=AlexNet()
alexnet(image_data)
alexnet.summary()

#start model anf train
total_dura=0.0
total_dura_squared=0.0

for step in range (num_batches+10):
    start_time=time.time()
    alexnet(image_data)
    duration=time.time()-start_time
    if step>=10:
        if step%10==0:
            printf('%s: step %d,duration=%.3f' % (datetime.now(),step-10,duration))
        total_dura += duration
        total_dura_squared+=duration*duration
average_time=total_dura/num_batches

print('%s:Forward across %d steps,%.3f +/- %.3f sec/batch'%(datetime.now(),num_batches,average_time,math.sqrt(total_dura_squared/num_batches-average_time*average_time)))

back_total_dura = 0.0
back_total_dura_squared = 0.0

for step in range(num_batches+10):
    start_time=time.time()

    with tf.GradientTape() as tape:
        loss=tf.nn.l2_loss(alexnet(image_data))
    gradients=tape.gradient(loss,alexnet.trainable_variables)
    duration=time.time()-start_time
    if step>=10:
        if step%10==0:
            printf('%s: step %d,duration=%.3f' % (datetime.now(),step-10,duration))
        back_total_dura += duration
        back_total_dura_squared+=duration*duration
back_agv_t=back_total_dura/num_batches

print('%s:Forward-backward across %d steps,%.3f +/- %.3f sec/batch'%(datetime.now(),num_batches,back_agv_t,math.sqrt(back_total_dura_squared/num_batches-back_agv_t*back_agv_t)))

9.3 VGGNet卷积网络模型
VGGNet模型是从图像中提取特征的CNN首选算法。
1.TensorFlow代码实现

##encode decode
import tensorflow as tf
from tensorflow.keras import layers
from datetime import datetime
import match
import time


batch_size=12
num_batches=100

#model model
w_init = tf.random_normal_initializer(stddev=1e-1)
b_init=tf.zeros_initializer()

#self_definited cnn
class Conv(layers.Layer):
    def __init__(self,layer,kernel_h,kernel_w,num_out,step_h,step_w):
        super(Conv,self).__init__()
        self.layer=layer
        self.k_h=kernel_h
        self.k_w=kernel_w
        self.num_out=num_out
        self.step_h=step_h
        self.step_w=step_w
   def build(self,input_shape):
        self.kernel_shape=[self.k_h,self.k_w,input_shape[-1],self.num_out]
        self.kernel=self.add_weight(name=self.layer+'/kernel',shape=self.kernel_shape,
                         initializer=w_init,dtype='float32',trainable=True)
        self.biases=self.add_weight(name=self.layer+'/biases',shape=[self.num_out],initializer=b_init,dtype='float32',trainable=True)
   def call(self,inputs):
        conv=tf.nn.conv2d(inputs,self.kernel,[1,self.step_h,self.step_w,1],padding="SAME")
        activation=tf.nn.relu(tf.nn.bias_add(conv,self.biases))
   return activation

class VGGNet(tf.keras.Model):
    def __init__(self,rate):
        super(VGGNet,self).__init__()

        self.conv1_1=Conv(layer="conv1_1",kernel_h=3,kernel_w=3,num_out=4,step_h=1,step_w=1)
        self.conv1_2=Conv(layer="conv1_2",kernel_h=3,kernel_w=3,num_out=64,step_h=1,step_w=1)
        self.pool1=layers.MaxPool2D(name="pool1",pool_size=(2,2),strides=(2,2),padding="SAME")

        self.conv2_1=Conv(layer="conv2_1",kernel_h=3,kernel_w=3,num_out=128,step_h=1,step_w=1)
        self.conv2_2=Conv(layer="conv2_2",kernel_h=3,kernel_w=3,num_out=128,step_h=1,step_w=1)
        self.pool2=layers.MaxPool2D(name="pool2",pool_size=(2,2),strides=(2,2),padding="SAME")

        self.conv3_1=Conv(layer="conv3_1",kernel_h=3,kernel_w=3,num_out=256,step_h=1,step_w=1)
        self.conv3_2=Conv(layer="conv3_2",kernel_h=3,kernel_w=3,num_out=256,step_h=1,step_w=1)
        self.conv3_3=Conv(layer="conv3_3",kernel_h=3,kernel_w=3,num_out=256,step_h=1,step_w=1)
        self.pool3=layers.MaxPool2D(name="pool3",pool_size=(2,2),strides=(2,2),padding="SAME")

        self.conv4_1=Conv(layer="conv4_1",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.conv4_2=Conv(layer="conv4_2",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.conv4_3=Conv(layer="conv4_3",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.pool4=layers.MaxPool2D(name="pool4",pool_size=(2,2),strides=(2,2),padding="SAME")

        self.conv5_1=Conv(layer="conv5_1",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.conv5_2=Conv(layer="conv5_2",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.conv5_3=Conv(layer="conv5_3",kernel_h=3,kernel_w=3,num_out=512,step_h=1,step_w=1)
        self.pool5=layers.MaxPool2D(name="pool5",pool_size=(2,2),strides=(2,2),padding="SAME")

        self.flatten=layers.Flatten()
        self.dense1=layers.Dense(name="full1",units=4096,activation='relu')
        self.dropout=layers.Dropout(rate=rate)
        self.dense2=layers.Dense(name="full2",uints=4096,activation='relu')
        self.dropout=layers.Dropout(rate=rate)
        self.dense3=layers.Dense(name="full3",units=1000,activation='softmax')
    def call(self,input):
        x=self.conv1_1(input)
        x=self.conv1_2(x)
        x=self.pool1(x)

        x=self.conv2_1(x)
        x=self.conv2_2(x)
        x=self.pool2(x)

        x=self.conv3_1(x)
        x=self.conv3_2(x)
        x=self.conv3_3(x)
        x=self.pool3(x)

        x=self.conv4_1(x)
        x=self.conv4_2(x)
        x=self.conv4_3(x)
        x=self.pool4(x)

        x=self.conv5_1(x)
        x=self.conv5_2(x)
        x=self.conv5_3(x)
        x=self.pool5(x)

        x=self.flatten(x)
        x=self.dense1(x)
        x=self.dropout(x)
        x=self.dense2(x)
        x=self.dropout(x)
        x=self.dense3(x)

        return tf.argmax(x,1)


#datasets
image_size=224
image_shape=[batch_size,image_size,image_size,3]
image_init=tf.random_normal_initializer(stddec=1e-1)
image_data=tf.Variale(initial_value=image_init(shape=image_shape),dtype='float32')

#print net message
vggnet=VGGNet(rate=0.5)
vggnet(image_data)
vggnet.summary()

#start model anf train
total_dura=0.0
total_dura_squared=0.0

for step in range (num_batches+10):
    start_time=time.time()
    vggnet(image_data)
    duration=time.time()-start_time
    if step>=10:
        if step%10==0:
            printf('%s: step %d,duration=%.3f' % (datetime.now(),step-10,duration))
        total_dura += duration
        total_dura_squared+=duration*duration
average_time=total_dura/num_batches

print('%s:Forward across %d steps,%.3f +/- %.3f sec/batch'%(datetime.now(),num_batches,average_time,math.sqrt(total_dura_squared/num_batches-average_time*average_time)))

back_total_dura = 0.0
back_total_dura_squared = 0.0

for step in range(num_batches+10):
    start_time=time.time()

    with tf.GradientTape() as tape:
        loss=tf.nn.l2_loss(alexnet(image_data))
    gradients=tape.gradient(loss,alexnet.trainable_variables)
    duration=time.time()-start_time
    if step>=10:
        if step%10==0:
            printf('%s: step %d,duration=%.3f' % (datetime.now(),step-10,duration))
        back_total_dura += duration
        back_total_dura_squared+=duration*duration
back_agv_t=back_total_dura/num_batches

print('%s:Forward-backward across %d steps,%.3f +/- %.3f sec/batch'%(datetime.now(),num_batches,back_agv_t,math.sqrt(back_total_dura_squared/num_batches-back_agv_t*back_agv_t)))

9.4 IncepinNet-v3卷积网络模型
1.模型结构

• 将全连接甚至一般的卷计划为稀疏链接
• 大规模稀疏网络可以通过分析激活值的统计特性和对高度相关的输出进行聚类来逐层构建处一个最优网络
• hebbian原理：神经反射活动的持续与重复会导致神经元链接稳定性的持久提升
• 采用MLPConv方式提高卷积层的表达能力，使用多层感知器替换单纯的卷积神经网络中的加权求和。
• 在此强推了**《Network in Network》**这篇论文

2.基于slim实现Inception-v3的最后一个module

##encode decode
import tensorflow as tf
import tensorflow.contrib.slim as slim

with slim.arg_scope([slim.conv2d,slim.max_pool2d.slim.avg_pool2d],stride=1,padding="SAME"):
    with tf.variable_scope("BRANCH_0"):
        branch_0=slim.conv2d(last_net,320,[1,1],scope="Cov2d_0a_1x1")
    with tf.variable_scope("BRANCH_1"):
        branch_1=slim.conv2d(last_net,384,[1,1],scope="Cov2d_1a_1x1")
        branch_1=tf.concat(3,[slim.conv2d(branch_1,384,[1,3],scope="Conv2d_1b_1x3"),
                              slim.conv2d(branch_1,384,[3,1],scope="Conv2d_1c_3x1")])
    
    with tf.variable_scope("BRANCH_2"):
        branch_2=slim.conv2d(last_net,448,[1,1],scope="Cov2d_2a_1x1")
        branch_2=slim.conv2d(last_net,384,[3,3],scope="Cov2d_2b_3x3")
        branch_2=tf.concat(3,[slim.conv2d(branch_2,384,[1,3],scope="Conv2d_2c_1x3"),
                              slim.conv2d(branch_2,384,[3,1],scope="Conv2d_2d_3x1")])


    with tf.variable_scope("BRANCH_3"):
        branch_3=slim.agv_pool2d(last_net,[3,3],scope="AvgPool_3a_3x3")
        branch_2=slim.conv2d(branch_3,192,[1,1],scope="Conv2d_3a_1x1")
        
    Module_output=tf.concat(3,[branch_0,branch_1,branch_2,branch_3])

3.使用Inception V3 完成模型迁徙

• 迁徙学习数据集下载：http://www.download.tensorflow.org/example_images/flower_photos.tgz
• 下载inception v3模型：
  wget https://storage.googleapis.com/download.tensorflow.org/models/inception_dec_2015.zip
• 图片处理函数flower_photos_dispose.py

##encode decode
import glob
import os.path
import random
import numpy as np
import tensorflow.python.platform import gfile

input_data="../flower_photos"
CACHE_DIR="../datasets/bottleneck"

##create_iamge_dict()
def create_iamge_dict():
    result={}
    path=[x[0] for x in os.walk(input_data)]
    is_root_dir=True
    for sub_dirs in path:
        if is_root_dir:
            is_root_dir=False
            continue
        extension_name=['.jpg','.jpeg','.JPG','.JPEG']
        images_list=[]
        for extension in extension_name:
            file_glob=os.path.join(sub_dirs,'*.'+extension)
            images_list.extend(glob.glob(file_glob))

        dir_name=os.path.basename(sub_dirs)
        flower_category=dir_name

        training_images=[]
        testing_images=[]
        validation_images=[]

        for image_name in images_list:
            image_name=os.path.basename(image_name)
            score=np.random.randient(100)
            if score<10:
                validation_images.append(image_name)
            elif score<20:
                testing_images.append(images_name)
            else:
                training_images.append(image_name)
        result[flower_category]={
        "dir":dir_name,
        "training":training_images,
        "testing":testing_images,
        "validation":validation_images,
         }
    return result

def get_random_bottlenecks(sess,num_classes,image_lists,batch_size,data_category,jpeg_data_tensor,bottleneck_tensor):
    bottlenecks=[]
    labels=[]

    for i in range(batch_size):
        random_index=random.randrange(num_classes)
        flower_category=list(image_lists.keys())[random_index]
        image_index=random.randrange(65536)
        bottlneck=get_or_create_bottleneck(sess,image_lists,flower_category,image_index,data_category,jpeg_data_tensor,bottleneck_tensor)
        label=np.zeros(num_classes,dtypes=np.float32)
        label[random_index]=1.0
        labels.append(label)
        bottlnecks.append(bottlneck)
    return bottlenecks,labels

def get_or_create_bottleneck(sess,image_lists,flower_category,image_index,data_category,jpeg_data_tensor,bottleneck_tensor):
    sub_dir=image_lists[flower_category]["dir"]
    sub_dir_path=os.path.join(CACHE_DIR,sub_dir)

    if not os.path.exists(sub_dir_path):
        os.makedirs(sub_dir_path)

    bottleneck_path=get_image_path(image_lists,CACHE_DIR,flower_category,image_index,data_category)+".txt"
    if not os.path.exists(bottleneck_path):
        image_path=get_image_path(image_lists,input_data,flower_category,image_index,data_category)
        image_data=gfile.FastGFile(image_path,"rb").read()

        bottleneck_values=sess.run(bottleneck_tensor,feed_dir={jpeg_data_tensor:image_data})
        bottleneck_values=np.squeeze(bottleneck_values)

        bottleneck_string=','.join(str(x) for x in bottleneck_values)
        with open(bottleneck_path,"w") as bottleneck_file:
            bottleneck_file.write(bottleneck_string)

    else:
        with open(bottleneck_path,"r") as bottleneck_file:
            bottleneck_string=bottleneck_file.read();

        bottleneck_values=[float(x) for x in bottleneck_string.split(',')]
    return bottleneck_values

def get_image_path(image_lists,image_dir,flower_category,image_index,data_category):
    category_list=image_lists[flower_category][data_category]
    actual_index=image_index%len(category_list)
    image_name=category_list[actual_index]
    sub_dir=image_lists[flower_category]["dir"]

    full_path=os.path.join(image_dir,sub_dir,image_name)
    return full_path

def get_test_bottlenecks(sess,image_lists,num_classes,jpeg_data_tensor,bottleneck_tensor):
    bottlenecks=[]
    labels=[]

    flower_category_list=list(image_lists.keys())
    data_category="testing"

    for label_index, flower_category in enumerate(flower_category_list):
        for image_index,unused_base_name in enumerate(image_lists[flower_category]["testing"]):
            bottleneck=get_or_create_bottleneck(sess,image_lists,flower_category,image_index,data_category,jpeg_data_tensor,bottleneck_tensor)

            label=np.zeros(num_classes,dtype=np.float32)
            label[label_index]=1.0
            labels.append(label)
            bottlenecks.append(bottleneck)
    return bottlenecks,labels

• InceptionV3.py

import tensorflow.compat.v1 as tf1
import os
import flower_photos_dispose as fd
from tensorflow.python.platform import gflie

model_path="../InceptionModel/inception_dec_2015/"
model_file="tensorflow_inception_graph.pb"

num_steps=4000
BATCH_SIZE=100

bottleneck_size=2048

image_lists=fd.create_image_dict()

num_clasees=len(image_lists.keys())

with gfile.FastGFile(os.path.join(model_path,model_file),'rb') as f:
    graph_def=tf1.GraphDef()
    graph_def.ParseFromString(f.read())

bottleneck_tensor,jpeg_data_tensor=tf1.import_graph_def(graph_def,return_elements=["pool_3/_reshape:0","DecodeJPeg/contents:0"])
tf1.disable_eager_execution()
x_=tf1.placeholder(tf1.float32,[None,bottleneck_size],name='BottleneckInpuPlaceholder')
y_=tf1.placeholder(tf1.float32,[None,num_clasees],name='GroundTruthInput')

with tf1.name_scope("final_training_ops"):
    weights=tf1.Variable(tf1.truncated_normal([bottleneck_size,num_clasees],stddev=0.001))
    biases=tf1.Variable(tf1.zeros([num_clasees]))
    logits=tf1.matmul(x,weights)+biases
    final_tensor=tf1.nn.softmax(logits)

cross_entropy=tf1.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_)
cross_entropy_mean=tf1.reduce_mean(cross_entropy)
train_step=td1.train.GradientDescentOptimizer(0.01).minimize(cross_entropy_mean)

correct_prediction=tf1.equal(tf1.argmax(final_tensor,1),tf1.argmax(y_,1))
evaluation_step=tf1.reduce_mean(tf1.cast(correct_prediction,tf1.float32))

with tf1.Session() as sess:
    init = tf1.global_variables_initializer()
    sess.run(init)
    for i in range(num_steps):
        train_bottlenecks,train_labels=fd.get_random_bottlenecks(sess,num_clasees,image_lists,BATCH_SIZE,"training",jpeg_data_tensor,bottleneck_tensor)
        sess.run(train_step,feed_dict={x:train_bottlenecks,y_:train_labels})

        if i% 100 == 0:
            validation_bottlenecks,validation_labels=fd.get_random_bottlenecks(sess,num_clasees,image_lists,BATCH_SIZE,"validation",jpeg_data_tensor,bottleneck_tensor)
            validation_accuracy=sess.run(evaluation_step,feed_dict={x:validation_bottlenecks,
                                                                    y_:validation_labels})
            print("Step %d: Validation accuracy = %.1f%%"%(i,validation_accuracy*100))

            test_bottlenecks,test_labels=fd.get_test_bottlenecks(sess,image_lists,num_clasees,jpeg_data_tensor,bottleneck_tensor)
            test_accuracy=sess.run(evaluation_step,feed_dict={x:test_bottlenecks,
                                                              y_:test_labels})
            print("Finally test accuracy = %.1f%%" % (test_accuracy*100))

9.5 ResNet卷积网络模型
引入残差学习单元（Residual Unit）
Highway NetWork：使得一定比例饿的前一层的信息没有经过矩阵惩罚和非线性变化而是直接传输到下一层。
1.TensorFlow代码实现

##encode decode
import collections
import tensorflow as tf
from tensorflow.keras import layers
from datetime import datetime
import math
import time

class Block(collections.namedtuple("block",["name","args"])):
    "A named tuple describing a ResNet Block"
    #namedtuple(typename,filed_names,verbose,rename)
blocks=[Block("block1",[(256,64,1),(256,64,1),(256,64,2)]),
        Block("block2",[(512,128,1)]*7+[(512,128,2)]),
        Block("block3",[(1024,256,1)]*35+[(1024,256,2)]),
        Block("block4",[(2048,512,1)]*3)
        ]
#
blocks_50=[Block("block1",[(256,64,1),(256,64,1),(256,64,2)]),
        Block("block2",[(512,128,1)]*3+[(512,128,2)]),
        Block("block3",[(1024,256,1)]*5+[(1024,256,2)]),
        Block("block4",[(2048,512,1)]*3)
        ]

blocks_101=[Block("block1",[(256,64,1),(256,64,1),(256,64,2)]),
        Block("block2",[(512,128,1)]*3+[(512,128,2)]),
        Block("block3",[(1024,256,1)]*22+[(1024,256,2)]),
        Block("block4",[(2048,512,1)]*3)
        ]

blocks_200=[Block("block1",[(256,64,1),(256,64,1),(256,64,2)]),
        Block("block2",[(512,128,1)]*23+[(512,128,2)]),
        Block("block3",[(1024,256,1)]*35+[(1024,256,2)]),
        Block("block4",[(2048,512,1)]*3)
        ]
class ResidualUnit(layers.Layer):
    def __init__(selfself,depth,depth_residual,stride):
        super(ResidualUnit,self).__init__()
        self.depth=depth
        self.depth_residual=depth_residual
        self.strides=stride
    def build(self,input_shape):
        self.depth_input=input_shape[-1]
        self.batch_normal=layers.BatchNormalization()
        self.identity_maxpool2d=layers.MaxPool2D(pool_size=(1,1),strides=self.stride)
        self.identity_conv2d=layers.Conv2D(filters=self.depth,kernal_size=[1,1],strides=self.stride,activation=None)

        self.conv1=layers.Conv2D(filters=self.depth_residual,kernel_size=[1,1],strides=self.stride,activation=None)

        self.conv_same=layers.Conv2D(filters=self.depth_residual,kernal_size=[3,3],strides=self.stride,padding='SAAME')
        self.conv_valid=layers.Conv2D(filters=self.depth_residual,kernal_size=[3,3],strides=self.stride,padding='VALID')
        self.conv3=layers.Conv2D(filters=self.depth,kernel_size=[1,1],strides=1,activation=None)

    def call(self,inputs):
        batch_norm=self.batch_normal(inputs)
        if self.depth==self.depth_input:
            if self.stride==1:
                identity=inputs
            else:
                identity=self.identity_maxpool2d(inputs)
        else:
            identity=self.identity_conv2d(inputs)
        residual=self.conv1(batch_norm)
        if self.stride == 1:
            residual=self.conv_same(residual)
        else:
            pad_begin=(3-1)//2
            pad_end=3-1-pad_begin
            residual=td.pad(residual,
                            [[0,0],[pad_begin,pad_end]],
                            [[pad_begin,pad_end],[0,0]])
            residual=self.conv_valid(residual)
        residual=self.conv3(residual)
        return identity+residual
class ResNet_v2(tf.keras.Model):
    def __init__(self,blocks):
        super(ResNet_v2,self).__init__()
        self.blocks=blocks

        #the first conv and pool cell=7x7,depth=64,pool_stride=3x3
        self.conv1=layers.Conv2D(filters=64,kernal_size=[7,7],strides=2)
        self.pool1=layers.MaxPool2D(pool_size=[3,3],strides=2)

        for block iin self.blocks:
            for i, tuple_value in enumerate(block.args):
                depth,depth_residual,stride=tuple_value
                setattr(self,block.name+“_”+str(i+1),ResidualUnit(depth,depth_residual,stride))
    def call(self,inputs):
        x=self.conv1(inputs)
        x=selff.pool1(x)
        for block in self.blocks:
            for i,tuple_value in enumerate(block.args):
                self.residualunit=getattr(self,block.name+"_"+str(i+1))
                x=self.residualunit(x)
        return x

resnetv2=ResNet_v2(blocks)
image_size = 224
image_shape = [batch_size, image_size, image_size, 3]
image_init = tf.random_normal_initializer(stddec=1e-1)
image_data = tf.Variale(initial_value=image_init(shape=image_shape), dtype='float32')
# print net message
resnetv2(image_data)
resnetv2.summary()
# start model anf train
total_dura = 0.0
total_dura_squared = 0.0
num_batches=100
for step in range(num_batches + 10):
    start_time = time.time()
    resnetv2(image_data)
    duration = time.time() - start_time
    if step >= 10:
        if step % 10 == 0:
            printf('%s: step %d,duration=%.3f' % (datetime.now(), step - 10, duration))
        total_dura += duration
        total_dura_squared += duration * duration
average_time = total_dura / num_batches

print('%s:Forward across %d steps,%.3f +/- %.3f sec/batch' % (datetime.now(), num_batches, average_time, math.sqrt(
    total_dura_squared / num_batches - average_time * average_time)))