pytorch加载自己的数据集，数据集载入-视频合集

pytorch加载数据主要学习了两种：只有图片的数据集和有scv保存标签的数据集 而第一种只有图片的数据集的加 载又分为两种：标签在文件夹上的和标签在图片名上的

1.第一种标签在文件夹上的数据加载方法 

输入：

import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import  numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")

”“”
完整程序：
data_transform = transforms.Compose([
 transforms.Resize(32), # 缩放图片(Image)，保持长宽比不变，最短边为32像素
 transforms.CenterCrop(32), # 从图片中间切出32*32的图片
 transforms.ToTensor(), # 将图片(Image)转成Tensor，归一化至[0, 1]
 transforms.Normalize(mean=[0.492, 0.461, 0.417], std=[0.256, 0.248, 0.251])
    # 标准化至[-1, 1]，规定均值和标准差
])

hymenoptera_dataset = datasets.ImageFolder\
    (root="F:\\jupyter\\pytorch\\data\\dogs_cats\\dataset\\train",
           transform=data_transform)
dataset_loader = torch.utils.data.DataLoader(hymenoptera_dataset,batch_size=4,shuffle=True)

“”“

data_transform=transforms.Compose([
    transforms.Resize(32),  # 缩放图片(Image)，保持长宽比不变，最短边为32像素
    transforms.CenterCrop(32),  # 从图片中间切出32*32的图片
    transforms.ToTensor(),  # 将图片(Image)转成Tensor，归一化至[0, 1]
    transforms.Normalize(mean=[0.492, 0.461, 0.417], std=[0.256, 0.248, 0.251])
    # 标准化至[-1, 1]，规定均值和标准差

])
"""
此函数的作用是对图片进行标准化和归一化 Resize(32)缩放图片(Image)，保持长宽比不变，
最短边为32像素 CenterCrop(32)从图片中间切出32*32的图片 
RandomSizedCrop(32)这一句的作用是对原图进行随机大小和高宽比的裁剪，最后的尺寸为32x32
RandomHorizontalFlip()这个则是对原图像根据概率进行随机水平翻转
transforms.ToTensor()将图片转化为张量，并使图片的形式表现为通道x高x宽的形式
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
这个则是对数据 进行正则化操作，第一个参数为均值，第二个参数为标准差。这两个值我是使用程序进行计算的。
代码如下
"""
import cv2
import os
img_h,img_w=32,32
means,stdevs=[],[]
img_list=[]
imgs_path=r"D:\caocao\hotdog\train"
imgs_path_list=os.listdir(imgs_path)
len_=len(imgs_path_list)
for item in imgs_path_list:
    img=cv2.imread(os.path.join(imgs_path,item))
    imgs=cv2.resize(img,(img_w,img_h))
    img=img[:,:,:,np.newaxis]
    img_list.append(img)
    i=i+1
    print(i,'/',len_)
imgs=np.concatenate(img_list,axis=3)
imgs=imgs.astype(np.float32)/255.
for i in range(3):
    pixels=imgs[:,:,i,:].ravel()
    means.append(np.mean(pixels))
    stdevs.append(np.std(pixels))
means.reverse()
stdevs.reverse()
print("normMean = {}".format(means))
print("normStd = {}".format(stdevs))


hymenoptera_dataset=datasets.ImageFolder(root="D:\caocao\hotdog\train",
                                         transforms=data_transform)
img,label=hymenoptera_dataset[0]
print(label)
print(img.size)
for img,label in hymenoptera_dataset:
    print("图像img的形状{},标签label的值{}".format(img.shape, label))
    print("图像数据预处理后：\n", img)
    break
dataset_loader=torch.utils.data.DataLoader(hymenoptera_dataset,batch_size=4,shuffle=True)

import torchvision
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline
# 显示图像
def imshow(img):
    img=img/2+0.5
    npimg=img.numpy()
    plt.imshow(np.transpose(npimg,(1,2,0)))
    plt.show()
dataiter=iter(dataset_loader)
images,labels=dataiter.next()
imshow(torchvision.utils.make_grid(images))
print(' '.join('%s' %["小狗" if labels[j].item()==1 else "小猫" for j in range(4)]))

2. 第二种标签在图片名上的数据加载方法

import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import  numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")

class MyDataset(Dataset):
    def __init__(self,path_dir,transform=None):
        self.path_dir=path_dir
        self.transform=transform
        self.images=os.listdir(self.path_dir)
    def __len__(self):
        return len(self.images)

    def __getitem__(self,index):
        image_index=self.images[index]
        img_path=os.path.join(self.path_dir,image_index)
        Image.open(img_path).convert("RGB")

        label=image_path.split('\\'')[-1].split('.')[0]
        label=1 if 'dog' in label else 0
        if self.transform is not None:
            img=self.transform(img)
        return img,label
from torchvision import transforms as T
        transform = T.Compose([
        T.Resize(32),  # 缩放图片(Image)，保持长宽比不变，最短边为224像素
        T.CenterCrop(32),  # 从图片中间切出224*224的图片
        T.ToTensor(),  # 将图片(Image)转成Tensor，归一化至[0, 1]
        T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])  # 标准化至[-1, 1]，规定均值和标准差
    ])

    dataset = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train", transform=transform)
    dataloader = DataLoader(dataset, batch_size=4, shuffle=True)

4.csv文件存储标签的数据集的加载的方法

输入：

#数据集的分割
import math
import os
import shutil
from collections import Counter
def reorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,
                   valid_ratio):
    # 读取训练数据标签。
    with open(os.path.join(data_dir, label_file), 'r') as f:
        # 跳过文件头行（栏名称）。
        lines = f.readlines()[1:]
        tokens = [l.rstrip().split(',') for l in lines]
        idx_label = dict(((idx, label) for idx, label in tokens))
    labels = set(idx_label.values())

    num_train = len(os.listdir(os.path.join(data_dir, train_dir)))
    # 训练集中数量最少一类的狗的数量。
    min_num_train_per_label = (
        Counter(idx_label.values()).most_common()[:-2:-1][0][1])
    # 验证集中每类狗的数量。
    num_valid_per_label = math.floor(min_num_train_per_label * valid_ratio)
    label_count = dict()

    def mkdir_if_not_exist(path):
        if not os.path.exists(os.path.join(*path)):
            os.makedirs(os.path.join(*path))

    # 整理训练和验证集。
    for train_file in os.listdir(os.path.join(data_dir, train_dir)):
        idx = train_file.split('.')[0]
        label = idx_label[idx]
        mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])
        shutil.copy(os.path.join(data_dir, train_dir, train_file),
                    os.path.join(data_dir, input_dir, 'train_valid', label))
        if label not in label_count or label_count[label] < num_valid_per_label:
            mkdir_if_not_exist([data_dir, input_dir, 'valid', label])
            shutil.copy(os.path.join(data_dir, train_dir, train_file),
                        os.path.join(data_dir, input_dir, 'valid', label))
            label_count[label] = label_count.get(label, 0) + 1
        else:
            mkdir_if_not_exist([data_dir, input_dir, 'train', label])
            shutil.copy(os.path.join(data_dir, train_dir, train_file),
                        os.path.join(data_dir, input_dir, 'train', label))

    # 整理测试集。
    mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])
    for test_file in os.listdir(os.path.join(data_dir, test_dir)):
        shutil.copy(os.path.join(data_dir, test_dir, test_file),
                    os.path.join(data_dir, input_dir, 'test', 'unknown'))

reorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,
                   valid_ratio)

#加载数据集
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'

from torch.utils.data import DataLoader,Dataset
import matplotlib.pyplot as plt
import torch
from torchvision import transforms, datasets
from PIL import Image
import pandas as pd
import numpy as np
#过滤警告信息
import warnings
warnings.filterwarnings("ignore")
transform_train = transforms.Compose([
    # 随机对图像裁剪出面积为原图像面积0.08~1倍、且高和宽之比在3/4~4/3的图像，再放缩为高和宽均为224像素的新图像
    transforms.RandomResizedCrop(28, scale=(0.08, 1.0),
                                 ratio=(3.0/4.0, 4.0/3.0)),
    # 以0.5的概率随机水平翻转
    transforms.RandomHorizontalFlip(),
    # 随机更改亮度、对比度和饱和度
    transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
    transforms.ToTensor(),
    # 对各个通道做标准化，(0.485, 0.456, 0.406)和(0.229, 0.224, 0.225)是在ImageNet上计算得的各通道均值与方差
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])  # ImageNet上的均值和方差
])

# 在测试集上的图像增强只做确定性的操作
transform_test = transforms.Compose([
    transforms.Resize(28),
    # 将图像中央的高和宽均为224的正方形区域裁剪出来
    transforms.CenterCrop(28),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
import torchvision
#然后开始加载数据集
new_data_dir="F:\\jupyter\\pytorch\\data\\dog-breed-identification\\train_valid_test"

train_ds = datasets.ImageFolder(root=os.path.join(new_data_dir, 'train'),transform=transform_train)
valid_ds = datasets.ImageFolder(root=os.path.join(new_data_dir, 'valid'),transform=transform_train)

train_loader = torch.utils.data.DataLoader(train_ds,batch_size=4,shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid_ds,batch_size=4,shuffle=True)

 3.

import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import  numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")
class MyDataset(Dataset):
    def __init__(self,root,transforms=None,train=True,test=False):
        self.test=test
        imgs=[os.path.join(root,img) for img in os.listdir(root)]
        if self.test:
            imgs=sorted(imgs,key=lambda x:int(x.split('.')[-2].split('\\')[-1]))
        else:
            imgs=sorted(imgs,key=lambda x:int(x.split('.')[-2]))
        imgs_num=len(imgs)
        if self.test:
            self.imgs=imgs
        elif train:
            self.imgs=imgs[:int(0.7*imgs_num)]
        else:
            self.imgs=imgs[int(0.7*imgs_num):]
        if transform is None:
            normalize=T.Normalize(mean=[0.488,0.455,0.417])
            if self.test or not train:
                self.transforms=T.Compose([
                    T.Resize(28),
                    T.CenterCrop(28),
                    T.ToTensor(),
                    normalize

                ])
            else:
                self.transforms= T.Compose([
                    T.Resize(28),
                    T.CenterCrop(28),
                    T.RandomHorizontalFlip(),
                    T.ToTensor(),
                    normalize
                ])
    def __len__(self):
        return len(self.imgs)
    def __getitem__(self, index):
        img_path=self.imgs[index]
        if self.test:
            label=int(self.imgs[index].split('.')[-2].split('\\')[-1])
        else:
            label=1 if 'dog' in img_path.split('\\')[-1] else 0
        data=Image.open(img_path)
        data=self.transforms(img)
        return data,label
train_data = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train",
                       train=True)
val_data = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train",
                     train=False)
'''
训练集train_data和验证集val_data加载数据时，需要将第二个参数train设置为对应的值，便可以获取对应的数据

由于dataset中直接默认进行了transforms操作所以如果需要获取未进行数据加强操作的dataseet
需要对transforms相关的操作进行注释掉即可。
#使用DataLoader加载训练集数据
train_dataloader = DataLoader(train_data,batch_size=4,shuffle=True)
for batch_datas, batch_labels in dataloader:
    print(batch_datas.size(),batch_labels.size())
    break


#使用DataLoader加载验证集数据
val_dataloader = DataLoader(val_data,batch_size=4,shuffle=True)
for batch_datas, batch_labels in dataloader:
    print(batch_datas.size(),batch_labels.size())
    break
'''

示例1：G:\deep-learn\deep-learning\代码总结\日月光华 数据集加载.py

输入：

image=glob.glob(r'dataset/images/*.jpg')
annotations=glob.glob(r'dataset/annotations/trimaps/*.png')

np.random.seed(2022)
index=np.random.permutation(len(images))

images=np.array(images)[index]
annotations=np.array(annotation)[index]

sep=int(len(images)*0.8)

train_imgs=images[:sep]
train_annos=annotations[:sep]
test_imgs=images[sep:]
test_annos=annotations[sep:]

#annotation 像素级 0,1,2,3

完整代码：

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

import torchvision
from torchvision import transforms
import os

import glob
from PIL import Image



image=glob.glob(r'dataset/images/*.jpg')
annotations=glob.glob(r'dataset/annotations/trimaps/*.png')

np.random.seed(2022)
index=np.random.permutation(len(images))

images=np.array(images)[index]
annotations=np.array(annotation)[index]

sep=int(len(images)*0.8)

train_imgs=images[:sep]
train_annos=annotations[:sep]
test_imgs=images[sep:]
test_annos=annotations[sep:]

#annotation 像素级 0,1,2,3

transform = transforms.Compose([
                    transforms.Resize((256, 256)),
                    transforms.ToTensor(),
])

class Oxford_dataset(data.Dataset):
    def __init__(self,img_path,anno_path):
        self.imgs=img_path
        self.annos=anno_path
    def __getitem__(self, item):
        img=self.imgs[item]
        anno=self.annos[item]
        pil_img=Image.open(img).convert('RGB')
        pil_tensor=transform(pil_img)

        pil_anno=Image.open(anno).resize((256,256))
        anno_tensor=torch.tensor(np.array(pil_anno),dtype=torch.int64)
        return img_tensor,anno_tensor-1 #1,2,3->0,1,2
    def __len__(self):
        return len(self.imgs)

train_dataset=Oxford_dataset(train_imgs,train_annos)
test_dataset=Oxford_dataset(test_imgs,test_annos)
batch_size=8
train_dl=data.DataLoader(train_dataset,batch_size=batch_size,
                         shuffle=True)
test_dl=data.DataLoader(test_dataset,batch_size=batch_size,
                         )

img_batch,anno_batch=next(iter(train_dl))


'''''''''''''''''''''文本分类'''''''''''''''''
train_iter,test_iter=torchtext.datasets.IMDB()

next(train_iter)
('neg',
 'I am ...')
#neg 标签，其他是内容

train_data,test_data=list(train_iter),list(test_iter)
all_classes=set([label for (label,text) in train_data])
#{'neg','pos'}

num_class=len(all_classes)
#导入分词工具
from torchtext.data.utils import get_tokenizer
tokenier=get_tokenizer('basic_english')
tokenier('This is a riyueguanghua Pytorch lesson')
#['this','is', 'a', 'riyueguanghua', 'pytorch', 'lesson']

from torchtext.vocab import build_vocab_from_iterator
#定义生成器，返回以分词的文本
def yield_tokens(data):
    for (_,text) in data:
        yield tokenier(text)

vocab=build_vocab_from_iterator(yield_tokens(train_data),
                                specials=['',''],
                                min_freq=3)
vocab['look']
#‘look 编码174
vocab['film']
# ‘film' 编码23
vocab.set_default_index(vocab[''])#不认识的单词默认编码为1
vocab['riyueguanghua']
#1

vocab(['the','file'])
#[2,7778]

inp='this is a riyue'
vocab(tokenier(inp))
#[14,10,6,1]

text_pipeline=lambda x:vocab(tokenier(x))
text_pipeline('this is a riyue')
#[14,10,6,1]

label_pipeline=lambda x:int(x=='pos')
label_pipeline('neg')
#0
label_pipeline('pos')
#1

from torch.utils.data import DataLoader

text_dl=DataLoader(train_data,
                   batch_size=64,
                   shuffle)
def collate_batch(text_label_batch):
    # text_label_batch返回一个元组，两个值
    # text_label_batch=next(train_iter)
    # ('neg',
    #  'I am ...'，
    #  )
    # # neg 标签，其他是内容。pos是标签，
    label_list,test_list,offset_list=[],[],[0]#第一个偏移量为0
    #offset_list保存每一条评论的长度,offset代表偏移值,
    #tensor(1,2,3,6,7,7,9,0,0,0),offset:0,,4,5>>>,每一条评论的起始位置
    for _label,_text in text_label_batch:
        label_list.append(label_pipeline(_label))
        precess_text=torch.tensor(text_pipeline(_text),
                                  dtype=torch.int64)
        text_list.append(precess_text)
        offset_list.append(precess_text.size(0))#0维
    label_list=torch.tensor(label_list)
    text_list=torch.cat(text_list)
    offsets=torch.tensor(offset_list[:-1]).cumsum(dim=0)
    return label_list.to(device),text_list.to(device),offsets

train_dl=data.DataLoader(train_dataset,batch_size=batch_size,
                         shuffle=True,
                         collate_fn=collate_batch())
test_dl=data.DataLoader(test_dataset,batch_size=batch_size,
                       collate_fn=collate_batch())