深度学习（二）：读取数据

首先，要先说明，我讲的这些深度学习实战项目流程，主要针对于自建数据集。

在使用前要导入需要用到的库，不然会导致代码报错。

读取数据

上边博文讲了怎么打标签，接下来就是读取已经打完标签的数据。

df = pd.read_csv('train.csv')
df = df.loc[:].values  #从多个维度（行和列）对读取所有数据
trainData = []
trainLabels = []
for item in df:
    trainLabels.append(item[0])  #先行再列地读取数据，item[0]=标签，即第一行第一个(索引为0)的值
    trainData.append(item[1:].reshape((300,100,1)))  #hwc，把原来的一行改为hwc格式满足keras的输入格式。

trainLabels = np_utils.to_categorical(trainLabels, 3)  # 类别
trainData = array(trainData)

构建网络进行训练，这里以lenet-5为例

model = Sequential()
model.add(Conv2D(filters=6, kernel_size=(5, 5), padding='same', input_shape=trainData.shape[1:], activation='tanh'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=16, kernel_size=(5, 5), padding='same', activation='tanh'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(120, activation='tanh'))
model.add(Dense(84, activation='tanh'))
model.add(Dense(3, activation='softmax'))
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(trainData, trainLabels, batch_size=64, epochs=1000, verbose=1, shuffle=True)
model.save('lenet5.h5')

在这里分享一个完整的打标签加上训练的代码。

import glob
import os
import cv2
import numpy as np
import random
import tensorflow as tf
from tensorflow import keras

tf.random.set_seed(520)
np.random.seed(520)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')


def Data_Generation():
    X_data = [];
    Y_data = []
    path_data = [];
    path_label = []

    # path_file=os.getcwd() #获取当前工作目录
    files = os.listdir('classification')  # 获取'pokemon'文件夹下的所有文件名

    for file in files:
        print(file)
        for path in glob.glob('classification/' + file + '/*.*'):
            if 'jpg' or 'png' or 'bmp' in path:  # 只获取jpg/png/bmp格式的图片
                path_data.append(path)

    random.shuffle(path_data)  # 打乱数据

    for paths in path_data:  # 
        if '类别名1' in paths:  # 为每一类打标签
            path_label.append(0)
        elif '类别名2' in paths:
            path_label.append(1)
        elif '类别名3' in paths:
            path_label.append(2)
        elif '类别名4' in paths:
            path_label.append(3)


        img = cv2.imread(paths)  # 用opencv读图片数据
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # cv的图片通道是BGR，要转换成送入NN的RGB
        img = cv2.resize(img, (78, 170))  # 统一图片大小
        X_data.append(img)

    L = len(path_data)
    Y_data = path_label
    X_data = np.array(X_data, dtype=float)
    Y_data = np.array(Y_data, dtype='uint8')
    X_train = X_data[0:int(L * 0.8)]  # 将数据分为训练集 验证集和测试集 比例为 0.8:0.1:0.1
    Y_train = Y_data[0:int(L * 0.8)]
    X_valid = X_data[int(L * 0.8):int(L * 0.9)]
    Y_valid = Y_data[int(L * 0.8):int(L * 0.9)]
    X_test = X_data[int(L * 0.9):]
    Y_test = Y_data[int(L * 0.9):]
    return X_train, Y_train, X_valid, Y_valid, X_test, Y_test, L


X_train, Y_train, X_valid, Y_valid, X_test, Y_test, L = Data_Generation()
np.savez(os.path.join('classification','data.npz'),X_train = X_train, Y_train = Y_train, X_valid = X_valid, Y_valid = Y_valid, X_test = X_test, Y_test = Y_test)
# 打包成npz的压缩格式 储存在工程文件目录中，这样运行程序进行测试时就不用每次都重复生成数据，直接调用npz就好

读取数据代码如下：

import numpy as np
 
data=np.load('data.npz')
 
X_train=data['X_train']
Y_train=data['Y_train']
X_valid=data['X_valid']
Y_valid=data['Y_valid']
X_test= data['X_test']
Y_test= data['Y_test']
 
print(X_train.shape,Y_train.shape)#shape：数量，width，height，channels
print(X_valid.shape,Y_valid.shape)
print(X_test.shape,Y_test.shape)

接下来训练网络

def normalize(x):
    img_mean = tf.constant([0.485, 0.456, 0.406])
    img_std = tf.constant([0.229, 0.224, 0.225])
    x = (x - img_mean)/img_std
    return x

batchsz=32
#print(shape(X_data), shape(Y_data))
#使用 tf.data.Dataset.from_tensor_slices 进行加载
train_db = tf.data.Dataset.from_tensor_slices((X_train,Y_train))
train_db = train_db.shuffle(10000).map(preprocess).batch(batchsz)
 #map预处理,预处理函数在上面有写
valid_db = tf.data.Dataset.from_tensor_slices((X_valid,Y_valid))
valid_db = valid_db.map(preprocess).batch(batchsz)
 
test_db = tf.data.Dataset.from_tensor_slices((X_test,Y_test))
test_db = test_db.map(preprocess).batch(batchsz)
 
net=keras.applications.DenseNet121(weights='imagenet',include_top=False,pooling='max')#这里使用了自带的DenseNet121网络 你也可以用keras.Sequential DIY模型
net.trainable=False
cnn_net=keras.Sequential([
    net,
    layers.Dense(1024,activation='relu'),
    layers.BatchNormalization(), #BN层 标准化数据
    layers.Dropout(rate=0.2),
    layers.Dense(4)])
#其要进行转换为array矩阵，其实际格式是（batch，height，width，C） 
cnn_net.build(input_shape=(4,32,32,3))
cnn_net.summary()
 
early_stopping=EarlyStopping(              #防止过拟合
    monitor='val_accuracy',
    min_delta=0.01,
    patience=3)
 
 
cnn_net.compile(optimizer=optimizers.Adam(lr=1e-3),
              loss=losses.CategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
history  = cnn_net.fit(train_db, validation_data=valid_db, validation_freq=1,                        epochs=50,callbacks=[early_stopping])
 
history = history.history #history字典类型，包含val_loss,val_acc,loss,acc四个key值
 
cnn_net.evaluate(test_db)
#训练结束以后保存model文件到本地方便做图片分类的时候直接调用
#way1 保存model成 .pb 格式 方便各个平台(移动端等)的调用
tf.saved_model.save(cnn_net,'densenet')  
#way2 保存model成 .h5格式 里面包含了模型结构和训练好的模型参数
cnn_net.save('densenet.h5')

test.py

import tensorflow as tf
from tensorflow import keras
import cv2
 
label=['0','1','2','3']
 
network = keras.models.load_model('densenet.h5')
network.summary()
 
image=cv2.imread('test.jpeg')
img=image.copy()
img=cv2.resize(img,(32,32))
img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
 
def normalize(x):
    img_mean = tf.constant([0.485, 0.456, 0.406])
    img_std = tf.constant([0.229, 0.224, 0.225])
    x = (x - img_mean)/img_std
    return x
 
def preprocess(x):
    x = tf.expand_dims(x,axis=0)
    x = tf.cast(x, dtype=tf.float32) / 255.
    x = normalize(x)
    return x
 
img=preprocess(img)
 
#img= tf.cast(img, dtype=tf.uint8)
 
result=network(img)
result=tf.nn.softmax(result)
 
index=tf.argmax(result,axis=-1)
print(label[int(index)])
 
 
cv2.putText(image,label[int(index)],(166,54),cv2.FONT_HERSHEY_SCRIPT_SIMPLEX, 1.2, (255,0,0),2)
cv2.imshow('img',image)
cv2.waitKey()
cv2.destroyAllWindows()