深度学习之paddle之卷积神经网络定义，自定义数据集

深度学习之paddle之卷积神经网络定义，自定义数据集

1. 定义卷积神经网络：

'''
定义卷积神经网络模型
'''
import paddle.nn as nn

class LeNet(nn.Layer):
    """
    继承paddle.nn.Layer定义网络结构
    """

    def __init__(self, num_classes=10):
        """
        初始化函数
        """
        super(LeNet, self).__init__()

        self.features = nn.Sequential(
            nn.Conv2D(in_channels=3, out_channels=6, kernel_size=3, stride=1, padding=1),  # 第一层卷积,卷积后特征图尺寸不变224×224 -> 224×224
            nn.ReLU(), # 激活函数
            nn.MaxPool2D(kernel_size=2, stride=2),  # 最大池化，下采样 224×224 -> 112×112 ， 深度不变 3->3 即通道数
            nn.Conv2D(in_channels=6, out_channels=16, kernel_size=5, stride=1, padding=0), # 第二层卷积 112×112 -> 108×108
            nn.ReLU(), # 激活函数
            nn.MaxPool2D(kernel_size=2, stride=2) # 最大池化，下采样 108×108 -> 54×54
        )

        self.fc = nn.Sequential(
            nn.Linear(16*54*54, 120),  # 全连接
            nn.Linear(120, 84),   # 全连接
            nn.Linear(84, num_classes) # 输出层
        )

    def forward(self, inputs):
        """
        前向计算
        """
        y = self.features(inputs)
        y = paddle.flatten(y, start_axis=1,stop_axis=-1)
        out = self.fc(y)

        return out

2. 自定义数据集：

import numpy as np
from PIL import Image
from paddle.io import Dataset
import paddle.vision.transforms as T
import paddle as pd

class MyDataset(Dataset):
    """
    步骤一：继承paddle.io.Dataset类
    """
    def __init__(self, datasetPath, size, transform):
        """
        步骤二：实现构造函数，定义数据读取方式，划分训练和测试数据集
        """
        super(MyDataset, self).__init__()
        self.file_list = datasetPath
        self.size=size
        self.transform=transform
    def __getitem__(self, index):  # 这个方法是必须要有的，用于按照索引读取每个元素的具体内容
        file_list=self.file_list
        with open(file_list, 'r') as f:
            lines = [line.strip() for line in f]
            for line in lines:
                img_path, lab = line.strip().split('\t')
                img = Image.open(img_path)
                img = img.convert("RGB")          
                img = np.array(img).astype('float32')                
                label = int(lab)
        if self.transform is not None:
            img = self.transform(img)
        return img, label
       
    def __len__(self):
        # 这个函数也必须要写，它返回的是数据集的长度，也就是多少张图片，要和loader的长度作区分
        return self.size

3. 网络训练：

import paddle.vision.transforms as T
import paddle
import warnings
from paddle.metric import Accuracy

warnings.filterwarnings("ignore")

transform = T.Compose([
    T.Resize([224, 224]),
    T.ToTensor(),
    T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    # T.Transpose(),
])

test_data_size= 105
train_data_size= 945

train_dataset = MyDataset("./train_data.txt",train_data_size,transform=transform)
test_dataset = MyDataset("./test_data.txt",test_data_size,transform=transform)

train_loader = paddle.io.DataLoader(train_dataset, places=paddle.CPUPlace(), batch_size=8, shuffle=True)
test_loader = paddle.io.DataLoader(test_dataset, places=paddle.CPUPlace(), batch_size=8, shuffle=True)

network = LeNet(num_classes=2)
# 模型封装
model = paddle.Model(network)

# 为模型训练做准备，参数optimizer设置优化器，参数loss损失函数，参数metrics设置精度计算方式
optim = paddle.optimizer.Adam(learning_rate=0.001, parameters=model.parameters())

# 配置模型
model.prepare(
    optim,
    paddle.nn.CrossEntropyLoss(),
    Accuracy(topk=(1, 2))
    )

model.fit(train_loader,
        #test_loader,
        epochs=10,
        verbose=1
        )

model.evaluate(test_loader, batch_size=8, verbose=1)


model.save('inference_model', False)

4. VOC数据集生成：

#导入所需的包
import os
import numpy as np
from PIL import Image
import paddle
import paddle.fluid as fluid
import paddle.fluid.layers as layers
from multiprocessing import cpu_count
from paddle.fluid.dygraph import Pool2D,Conv2D
from paddle.fluid.dygraph import Linear


# 生成图像列表
data_path = './dataset'
if(os.path.exists('./train_data.txt')):
    os.remove('./train_data.txt')
if(os.path.exists('./test_data.txt')):
    os.remove('./test_data.txt')
    

dirName = data_path
train_data_size=0
test_data_size=0

with open('./train_data.txt', 'a') as f_train:
        with open('./test_data.txt', 'a') as f_test:
            list = os.listdir(dirName)  # 列出文件夹下所有的目录与文件
            count = 0
            for i in range(0, len(list)):
                path = os.path.join(dirName,list[i])
                text = path
                if os.path.isfile(path):                        
                    if list[i].split('-')[0] == 'neg':
                        text = text + '\t' + '0' + '\n'
                    else:
                        text = text + '\t' + '1' + '\n'
                if count%10 == 0:
                    f_test.write(text)
                    test_data_size += 1
                else:
                    f_train.write(text)
                    train_data_size += 1
                count += 1
print('列表已生成')
print("test_data_size=",test_data_size)
print("train_data_size=",train_data_size)

5. HOG+SVM 分类

! pip install scikit-image
import cv2
import os
import numpy as np
import random
from PIL import Image
from skimage.feature import hog
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
import pandas as pd

# 加载样本并计算HOG特征值 得到数据集
def loadImageHOGDataset(train_data, test_data):
    train_x=[]
    train_y=[]
    test_x=[]
    test_y=[]
    # train_data
    with open(train_data, 'r') as f:
            lines = [line.strip() for line in f]
            for line in lines:
                img_path, lab = line.strip().split('\t')
                tempImage = cv2.imread(img_path)
                image = cv2.resize(tempImage,(64,128))        
                # 计算HOG特征
                fd, hog_image = hog(image, orientations=9, pixels_per_cell=(8, 8),
                        cells_per_block=(2, 2), visualize=True, multichannel=True)     
                # 各参数说明
                # orientations是所创建的bucket数量。因为本文中想得到一个9x1的矩阵，所以将orientations设置为9。
                # pixels_per_cell定义所创建直方图的单元尺寸
                # cells_per_block指所规范化的直方图上的小块区域大小
                # 函数的特征矩阵存储在变量fd中, 图像存储于hog_image中
                # ‘visualize = True’，它将返回一张HOG的图像         
                label = int(lab)
                train_x.append(fd)
                train_y.append(label)
    # test_data
    with open(test_data, 'r') as f:
            lines = [line.strip() for line in f]
            for line in lines:
                img_path, lab = line.strip().split('\t')
                tempImage = cv2.imread(img_path)
                image = cv2.resize(tempImage,(64,128))        
                # 计算HOG特征
                fd, hog_image = hog(image, orientations=9, pixels_per_cell=(8, 8),
                        cells_per_block=(2, 2), visualize=True, multichannel=True)     
                # 各参数说明
                # orientations是所创建的bucket数量。因为本文中想得到一个9x1的矩阵，所以将orientations设置为9。
                # pixels_per_cell定义所创建直方图的单元尺寸
                # cells_per_block指所规范化的直方图上的小块区域大小
                # 函数的特征矩阵存储在变量fd中, 图像存储于hog_image中
                # ‘visualize = True’，它将返回一张HOG的图像         
                label = int(lab)
                test_x.append(fd)
                test_y.append(label)
    return train_x, test_x, train_y, test_y


# SVM 分类训练并预测
#加载数
train_x,test_x,train_y,test_y = loadImageHOGDataset(train_data="./train_data.txt", test_data="./test_data.txt")
print("训练集：",len(train_y))
print("验证集：",len(test_y))
clf = LinearSVC(C=1.0, loss="hinge", max_iter=10000)
clf.fit(np.array(train_x),np.array(train_y))
#使用支持向量机SVM分类器进行预测
y_pred = clf.predict(np.array(test_x))
print("支持向量机SVM模型评估结果：")
print("准确率=",accuracy_score(y_pred,test_y))
#使用3折交叉验证示例（不含验证集）
print("交叉验证比=",cross_val_score(clf, train_x, train_y, cv=3))
#计算混淆矩阵
print("混淆矩阵=")
print(confusion_matrix(test_y, y_pred, labels=None, sample_weight=None))