利用kaggle+pytorch进行机器学习1（图像分类）

最近在学习机器学习，一开始准备适应tensorflow框架，结果学习的时候发现tensrflow2.0与1.0版本兼容性太差，于是采用pytorch框架。
对于学生党而言，免费的GPU是最重要的资源，kaggle提供了每周30个小时的免费GPU，基本满足了学习需要。
没有自己写源码，参考了kaggle上面的一个资源，并作了一定注释和修改。

#导入必要的包
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import torch
import torch.nn as nn
import cv2
import matplotlib.pyplot as plt
import torchvision
from torch.utils.data import Dataset, DataLoader, ConcatDataset
from torchvision import transforms
import copy
import tqdm
from PIL import Image

%matplotlib inline

在这里插入代码片

#图片文件解压缩
!unzip ../input/train -d train
!unzip ../input/test1 -d test1

#设置文件路径
train_dir = '/kaggle/working/train/train'
test_dir = '/kaggle/working/test1/test1'
train_files = os.listdir(train_dir)
test_files = os.listdir(test_dir)

#图片处理类
class CatDogDataset(Dataset):
    def __init__(self, file_list, dir, mode='train', transform = None):
        self.file_list = file_list
        self.dir = dir
        self.mode= mode
        self.transform = transform
        if self.mode == 'train':
            if 'dog' in self.file_list[0]:
                self.label = 1
            else:
                self.label = 0
            
    def __len__(self):
        return len(self.file_list)
    
    def __getitem__(self, idx):
        img = Image.open(os.path.join(self.dir, self.file_list[idx]))
        if self.transform:
            img = self.transform(img)
        if self.mode == 'train':
            img = img.numpy()
            return img.astype('float32'), self.label
        else:
            img = img.numpy()
            return img.astype('float32'), self.file_list[idx]
  #数据转换函数
data_transform = transforms.Compose([
    transforms.Resize(256),
    transforms.ColorJitter(),
    transforms.RandomCrop(224),
    transforms.RandomHorizontalFlip(),
    transforms.Resize(128),
    transforms.ToTensor()
])
#分离猫，狗的图像数据
cat_files = [tf for tf in train_files if 'cat' in tf]
dog_files = [tf for tf in train_files if 'dog' in tf]

cats = CatDogDataset(cat_files, train_dir, transform = data_transform)
dogs = CatDogDataset(dog_files, train_dir, transform = data_transform)

catdogs = ConcatDataset([cats, dogs])

#数据加载
dataloader = DataLoader(catdogs, batch_size = 32, shuffle=True, num_workers=4)
samples, labels = iter(dataloader).next()
plt.figure(figsize=(16,24))
grid_imgs = torchvision.utils.make_grid(samples[:24])
np_grid_imgs = grid_imgs.numpy()
# in tensor, image is (batch, width, height), so you have to transpose it to (width, height, batch) in numpy to show it.
plt.imshow(np.transpose(np_grid_imgs, (1,2,0)))
# transfer learning
#使用Gpu
device = 'cuda'
model = torchvision.models.densenet121(pretrained=True) #调用pytorch中的预训练模型

num_ftrs = model.classifier.in_features
model.classifier = nn.Sequential(
    nn.Linear(num_ftrs, 500),
    nn.Linear(500, 2) #二分类，最后一层的输出改为2
)

model = model.to(device)
criterion = nn.CrossEntropyLoss() #损失函数
optimizer = torch.optim.Adam(model.parameters(), lr=0.002, amsgrad=True) #优化器
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[500,1000,1500], gamma=0.5) #调整学习率，增加随机性

epochs = 3
itr = 1
p_itr = 200
model.train()
total_loss = 0
loss_list = []
acc_list = []
for epoch in range(epochs):
    for samples, labels in dataloader:
        samples, labels = samples.to(device), labels.to(device)
        optimizer.zero_grad() #梯度参数清零
        output = model(samples) #前向传播
        loss = criterion(output, labels) #损失值
        loss.backward() #反向优化
        optimizer.step() #更新模型
        total_loss += loss.item() #统计
        scheduler.step()
        
        if itr%p_itr == 0: #两百张图片输出一次
            pred = torch.argmax(output, dim=1)#返回指定维度最大值序号，这里维度为1
            correct = pred.eq(labels)
            acc = torch.mean(correct.float())
            print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, Accuracy: {:.3f}'.format(epoch+1, epochs, itr, total_loss/p_itr, acc))
            loss_list.append(total_loss/p_itr)
            acc_list.append(acc)
            total_loss = 0
            
        itr += 1
#绘制损失值和准确度的变化图形
plt.plot(loss_list, label='loss')
plt.plot(acc_list, label='accuracy')
plt.legend()
plt.title('training loss and accuracy')
plt.show()

filename_pth = 'ckpt_densenet121_catdog.pth'
torch.save(model.state_dict(), filename_pth)#保存模型

test_transform = transforms.Compose([
    transforms.Resize((128,128)),
    transforms.ToTensor()
])

testset = CatDogDataset(test_files, test_dir, mode='test', transform = test_transform) #加载测试数据
testloader = DataLoader(testset, batch_size = 32, shuffle=False, num_workers=4)

#测试模型
model.eval()#固定模型权值
fn_list = []
pred_list = []
for x, fn in testloader:
    with torch.no_grad():
        x = x.to(device)
        output = model(x)
        pred = torch.argmax(output, dim=1)
        fn_list += [n[:-4] for n in fn]
        pred_list += [p.item() for p in pred]

submission = pd.DataFrame({"id":fn_list, "label":pred_list})
submission.to_csv('preds_densenet121.csv', index=False)#保存结果
    

#预测结果展示
samples, _ = iter(testloader).next()
samples = samples.to(device)
fig = plt.figure(figsize=(24, 16))
fig.tight_layout()
output = model(samples[:24])
pred = torch.argmax(output, dim=1)
pred = [p.item() for p in pred]
ad = {0:'cat', 1:'dog'}
for num, sample in enumerate(samples[:24]):
    plt.subplot(4,6,num+1)
    plt.title(ad[pred[num]])
    plt.axis('off')
    sample = sample.cpu().numpy()
    plt.imshow(np.transpose(sample, (1,2,0)))

下一篇文章，我将动手搭建一个经典CNN网络，来进行同样的二分类问题！