Resnet18-cifar10及Million-AID数据加载

---

---

一、Resnet18-cifar10

CIFAR-10 数据集由 10 类中的 60000 张 32x32 彩色图像组成，每类 6000 张图像。有 50000 张训练图像和 10000 张测试图像。

数据集分为五个训练批次和一个测试批次，每个批次有 10000 张图像。测试批次包含来自每个类的 1000 个随机选择的图像。训练批次以随机顺序包含剩余的图像，但某些训练批次可能包含来自一个类的图像多于另一个类的图像。在它们之间，训练批次正好包含来自每个类的 5000 张图像。

#  作者：暮看云
#  开发时间：2022/11/22 10:15

import torch as t
import torch.nn as nn
import numpy as np
from torch.autograd import Variable
from torch.nn import functional as F
import torchvision as tv
import torchvision.transforms as transforms
from torchvision.transforms import ToPILImage
show = ToPILImage()
from torch import optim
from torch.utils.data import DataLoader
import torchvision.models as models

transform = transforms.Compose([
    transforms.Resize([64,64]),  # 转换为64*64
    transforms.ToTensor(),   # 转换为Tensor格式
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),# 标准化
    # 数据预处理
])
a = t.utils.data.dataset
trainset = tv.datasets.CIFAR10(
    root='D:\Paper\Dataset\cifar-10-python',
    # 服务器路径：/home/shaowenjie/Dataset
    # 本地路径：D:\Paper\Dataset\cifar-10-python
    train=True,
    download=False,
    transform=transform,
)
# 生成训练数据集

trainloader = t.utils.data.DataLoader(
    trainset,
    batch_size=4,
    shuffle=True,
    num_workers=2,
)
# 加载训练数据集，一个batch大小为4

testset = tv.datasets.CIFAR10(
    'D:\Paper\Dataset\cifar-10-python',
    train=False,
    download=False,
    transform=transform,
                              )
# 生成测试数据集        

testloader = t.utils.data.DataLoader(
    testset,
    batch_size=4,
    shuffle=False,
    num_workers=2,
)
# 加载测试数据集
print(1)


class LinearClassifierNet(nn.Module):

    def __init__(self):
        super(LinearClassifierNet, self).__init__()
        self.fc = nn.Linear(3072,10)

    def forward(self,x):
        x = x.flatten(1, 3)
        output = self.fc(x)
        return output
# 简单线性分类器
class FCNet(nn.Module):
    def __init__(self):
        super(FCNet, self).__init__()
        self.fc1 = nn.Sequential(nn.Linear(3072,512),
                                 nn.ReLU(True))
        self.fc2 = nn.Sequential(nn.Linear(512,64),
                                 nn.ReLU(True))
        self.fc3 = nn.Linear(64,10)

    def forward(self,x):
        x = x.flatten(1,3)
        output = self.fc1(x)
        output = self.fc2(output)
        output = self.fc3(output)
        return output
# 三层全连接神经网络

class NetWork(nn.Module):

    def __init__(self):
        super(NetWork, self).__init__()
        self.model=nn.Sequential(
            nn.Conv2d(3,32,5,1,2),
            nn.MaxPool2d(2),
            nn.Conv2d(32,32,5,1,2),
            nn.MaxPool2d(2),
            nn.Conv2d(32,64,5,1,2),
            nn.MaxPool2d(2),
            nn.Flatten(),
            nn.Linear(64*4*4,64),
            nn.Linear(64,10)
        )#在序列中去填网络结构

    def forward(self,x):
        x = self.model(x)
        return x
# 普通卷积神经网络


class LeNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)

        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(x.size()[0], -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
# Lenet神经网络
    
    
device1 = t.device('cuda:0' if t.cuda.is_available() else 'cpu')  # 使用gpu运算
device2 = t.device('cpu')
net = models.resnet18(num_classes = 10)

net.to(device1)
criterion = nn.CrossEntropyLoss()   # 损失函数
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)  # 优化器
if __name__ == '__main__':
    for epoch in range(1):
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):#enumerate()用来遍历一个集合对象，它在遍历的同时还可以得到当前元素的索引位置。
            inputs, labels = data
            inputs, labels = Variable(inputs), Variable(labels)
            inputs = inputs.to(device1)
            labels = labels.to(device1)
            optimizer.zero_grad()

            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
            if i % 500 == 499:
                print('[%d, %5d] loss: %.3f' \
                      % (epoch + 1, i+1, running_loss / 500))
                running_loss = 0.0
            #  训练过程
    print('Finished Training')
    print('start test')
    correct = 0
    total = 0
    for data in testloader:
        images, labels = data
        images = images.to(device1)
        labels = labels.to(device1)
        outputs = net(Variable(images))
        _, predicted = t.max(outputs, dim=1)# 得到分类类别
        total += labels.size(0)
        correct += (predicted == labels).sum()
    # 测试过程
    print(str(100 * int(correct)/(total))+'%')
    # 输出分类准确度

搭建了几个简单的网络，以及使用torchvision.models中预训练好的网络，对cifar10数据集进行分类任务的训练和测试。

二、Million-AID数据加载

import os
import torch
from PIL import Image
import os
import numpy as np
from torchvision import transforms as TRAN
from torch.utils import data
from torch.autograd import Variable
import matplotlib.pyplot as plt
import matplotlib.image as ig
transforms = TRAN.Compose([
    TRAN.Resize(256),  # 转换为大小256*256
    TRAN.CenterCrop(256),# 中心裁剪256*256
    TRAN.ToTensor(),
    TRAN.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])# 标准化
])


class MillionAIDTest(data.Dataset):

    def __init__(self, Imageroot):
        self.imagespath = Imageroot
        self.transforms = transforms

    def __getitem__(self):
        datas = []
        for i,Imageroot in enumerate(self.imagespath):
            images = os.listdir(Imageroot)
            # 获得加载的训练集所有图片路径
            absimage = [os.path.join(Imageroot, image) for image in images]
            #  将图片所在路径与图片名拼接，得到绝对路径
            length = len(absimage)
            for index in range(length):
                image_path = absimage[index]
            # 这里只加载了Million-AID中农业用地-耕种用地-干燥土地一类，因此简单的置标签为1
                data = Image.open(image_path)
            # 图片读取
                if self.transforms:
                    data = self.transforms(data)
                datas.append(data)
        return datas


class MillionAIDTrain(data.Dataset):

    def __init__(self, Imageroot):
        self.imagespath = Imageroot
        self.transforms = transforms

    def __getitem__(self):
        datas = []
        labels = []
        for lable,Imageroot in enumerate(self.imagespath):
            images = os.listdir(Imageroot)
            # 获得加载的训练集所有图片路径
            absimage = [os.path.join(Imageroot, image) for image in images]
            #  将图片所在路径与图片名拼接，得到绝对路径
            length = len(absimage)
            lab = lable
            for index in range(length):
                image_path = absimage[index]
                data = Image.open(image_path)
            # 图片读取
                if self.transforms:
                    data = self.transforms(data)
                datas.append(data)
                labels.append(lab)

        return datas,labels


def main():
    rootstrain = ['D:\\Paper\\Dataset\\Million-AID\\train\\train\\train\\train\\agriculture_land\\arable_land\\dry_field'
        ,'D:\Paper\Dataset\Million-AID\\train\\train\\train\\train\\agriculture_land\\arable_land\greenhouse']
    # 训练集数据目录
    rootstest = ['D:\\Paper\\Dataset\\Million-AID\\test001\\test']
    # 测试集数据目录
    Datasettrain= MillionAIDTrain(rootstrain)
    datasettrain,lable = Datasettrain.__getitem__()
    lable = torch.from_numpy(np.array(lable,dtype=float))
    Datesettest = MillionAIDTest(rootstest)
    datasettest = Datesettest.__getitem__()

if __name__ == '__main__':
    main()

由于Million-AID数据集的训练集以文件目录名作为分类标签，因此，须转换为数字标签，且只考虑第三级51个类别，重写了MillionAID数据集的加载，返回可用于训练的数据和标签。

---

总结

本文简单介绍了基于cifar10数据集的深度学习分类任务，并对遥感数据集Million-AID进行处理，得到便于使用的数据和标签。