Pytorch中自定义Dataset读取数据

项目文件：

custom_dataset
├─ main.py      
├─ my_dataset.py
└─ utils.py

数据集

使用pytorch搭建AlexNet并训练花分类数据集

1.model.py

import os

import torch
from torchvision import transforms

from my_dataset import MyDataSet
from utils import read_split_data, plot_data_loader_image

# http://download.tensorflow.org/example_images/flower_photos.tgz
root = "D:/Dataset/flower_photos"  # 数据集所在根目录


def main():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")         #判断当前设备是否有GPU，有就用没有就不用
    print("using {} device.".format(device))                                      #打印使用的设备信息

    train_images_path, train_images_label, val_images_path, val_images_label = read_split_data(root)  #传入数据集，通过read_split_data方法划分训练集和验证集

    data_transform = {                                                             #定义训练集和验证集预处理方法
        "train": transforms.Compose([transforms.RandomResizedCrop(224),            #随机裁剪到224x224
                                     transforms.RandomHorizontalFlip(),            #随机水平翻转
                                     transforms.ToTensor(),                        #转化成Tensor格式
                                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),   #标准化处理
        "val": transforms.Compose([transforms.Resize(256),                         #将最小边resize到256x256
                                   transforms.CenterCrop(224),                     #进行中心裁剪到224x224
                                   transforms.ToTensor(),
                                   transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])}

    train_data_set = MyDataSet(images_path=train_images_path,                #训练集图像的路径列表
                               images_class=train_images_label,              #训练集图像的标签信息
                               transform=data_transform["train"])            #预处理方法

    batch_size = 8
    nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8])  # number of workers
    print('Using {} dataloader workers'.format(nw))
    train_loader = torch.utils.data.DataLoader(train_data_set,
                                               batch_size=batch_size,
                                               shuffle=True,            #shuffle=True在训练过程中会打乱数据集的顺序
                                               num_workers=nw,
                                               collate_fn=train_data_set.collate_fn)

    # plot_data_loader_image(train_loader)

    for step, data in enumerate(train_loader):
        images, labels = data


if __name__ == '__main__':
    main()

2.my_dataset.py

from PIL import Image
import torch
from torch.utils.data import Dataset


class MyDataSet(Dataset):
    """自定义数据集"""

    def __init__(self, images_path: list, images_class: list, transform=None):     #初始化函数
        self.images_path = images_path
        self.images_class = images_class
        self.transform = transform

    def __len__(self):                              #计算该数据集下所有样本个数
        return len(self.images_path)

    def __getitem__(self, item):                    #每次传入索引就返回该索引所对应的图片和标签信息
        img = Image.open(self.images_path[item])
        # RGB为彩色图片，L为灰度图片
        if img.mode != 'RGB':              #如果是其他格式的图片，可以将'RGB'更改
            raise ValueError("image: {} isn't RGB mode.".format(self.images_path[item]))
        label = self.images_class[item]

        if self.transform is not None:
            img = self.transform(img)

        return img, label

    @staticmethod
    def collate_fn(batch):                     #
        # 官方实现的default_collate可以参考
        # https://github.com/pytorch/pytorch/blob/67b7e751e6b5931a9f45274653f4f653a4e6cdf6/torch/utils/data/_utils/collate.py
        images, labels = tuple(zip(*batch))

        images = torch.stack(images, dim=0)
        labels = torch.as_tensor(labels)
        return images, labels

3.utils.py

import os
import json
import pickle
import random

import matplotlib.pyplot as plt


def read_split_data(root: str, val_rate: float = 0.2):             #两个参数：数据集路径、验证集所占数据集的比例
    random.seed(0)  # 将随机种子设置为0，保证随机结果可复现
    assert os.path.exists(root), "dataset root: {} does not exist.".format(root)   #判断数据集路径是否存在

    # 遍历文件夹，一个文件夹对应一个类别      通过os.path.join()将路径和遍历得到的文件名称拼接，然后判断是否是文件夹
    flower_class = [cla for cla in os.listdir(root) if os.path.isdir(os.path.join(root, cla))]   #通过os.listdir()遍历路径下的文件夹或文件，如果路径下的时文件夹就将其保存，如果时文件就丢弃
    # 排序，保证顺序一致
    flower_class.sort()
    # 生成类别名称以及对应的数字索引
    class_indices = dict((k, v) for v, k in enumerate(flower_class))
    json_str = json.dumps(dict((val, key) for key, val in class_indices.items()), indent=4)
    with open('class_indices.json', 'w') as json_file:
        json_file.write(json_str)

    train_images_path = []   # 存储训练集的所有图片路径
    train_images_label = []  # 存储训练集图片对应索引信息
    val_images_path = []     # 存储验证集的所有图片路径
    val_images_label = []    # 存储验证集图片对应索引信息
    every_class_num = []     # 存储每个类别的样本总数
    supported = [".jpg", ".JPG", ".png", ".PNG"]  # 支持的文件后缀类型
    # 遍历每个文件夹下的文件
    for cla in flower_class:
        cla_path = os.path.join(root, cla)
        # 遍历获取supported支持的所有文件路径
        images = [os.path.join(root, cla, i) for i in os.listdir(cla_path)
                  if os.path.splitext(i)[-1] in supported]                        #判断是否是支持的文件类型
        # 获取该类别对应的索引
        image_class = class_indices[cla]
        # 记录该类别的样本数量
        every_class_num.append(len(images))
        # 按比例随机采样验证样本
        val_path = random.sample(images, k=int(len(images) * val_rate))

        for img_path in images:
            if img_path in val_path:  # 如果该路径在采样的验证集样本中则存入验证集
                val_images_path.append(img_path)
                val_images_label.append(image_class)
            else:  # 否则存入训练集
                train_images_path.append(img_path)
                train_images_label.append(image_class)

    print("{} images were found in the dataset.".format(sum(every_class_num)))
    print("{} images for training.".format(len(train_images_path)))
    print("{} images for validation.".format(len(val_images_path)))

    plot_image = False                  #是否绘制每个类别的目标个数，设置为True后在下边return中打断点
    if plot_image:
        # 绘制每种类别个数柱状图
        plt.bar(range(len(flower_class)), every_class_num, align='center')
        # 将横坐标0,1,2,3,4替换为相应的类别名称
        plt.xticks(range(len(flower_class)), flower_class)
        # 在柱状图上添加数值标签
        for i, v in enumerate(every_class_num):
            plt.text(x=i, y=v + 5, s=str(v), ha='center')
        # 设置x坐标
        plt.xlabel('image class')
        # 设置y坐标
        plt.ylabel('number of images')
        # 设置柱状图的标题
        plt.title('flower class distribution')
        plt.show()

    return train_images_path, train_images_label, val_images_path, val_images_label


def plot_data_loader_image(data_loader):
    batch_size = data_loader.batch_size
    plot_num = min(batch_size, 4)

    json_path = './class_indices.json'
    assert os.path.exists(json_path), json_path + " does not exist."
    json_file = open(json_path, 'r')
    class_indices = json.load(json_file)

    for data in data_loader:
        images, labels = data
        for i in range(plot_num):
            # [C, H, W] -> [H, W, C]
            img = images[i].numpy().transpose(1, 2, 0)
            # 反Normalize操作
            img = (img * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]) * 255
            label = labels[i].item()
            plt.subplot(1, plot_num, i+1)
            plt.xlabel(class_indices[str(label)])
            plt.xticks([])  # 去掉x轴的刻度
            plt.yticks([])  # 去掉y轴的刻度
            plt.imshow(img.astype('uint8'))
        plt.show()


def write_pickle(list_info: list, file_name: str):
    with open(file_name, 'wb') as f:
        pickle.dump(list_info, f)


def read_pickle(file_name: str) -> list:
    with open(file_name, 'rb') as f:
        info_list = pickle.load(f)
        return info_list

感觉自己理解的不是很透彻
有兴趣的同学可以去看在pytorch中自定义dataset读取数据