data_loader.py
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from torch.utils.data.dataset import Dataset
from torchvision import transforms
import torch
class CustomDatasetFromCSV(Dataset):
def __init__(self,csv_path, height, width, transforms=None):
"""
:param csv_path:
:param height:
:param width:
:param transforms:
:param train: True is train data
"""
self.data = pd.read_csv(csv_path)
# print("self.data:\n", self.data)
self.labels = np.array(self.data["label"])
# print("self.labels:", self.labels)self.labels: [1 0 1 ... 7 6 9]
#print("self.labels.shape:", self.labels.shape)#self.labels.shape: (42000,)
self.height = height
self.width = width
self.transforms = transforms
def __getitem__(self, index):
single_image_label = self.labels[index]
img_as_np = np.array(self.data.loc[index][1:]).reshape(self.height,self.width).astype("uint8")
if self.transforms is not None:
img_as_tensor = self.transforms(img_as_np)
return (img_as_tensor, single_image_label)
def __len__(self):
return len(self.data.index)
class Test_CustomDatasetFromCSV(Dataset):
def __init__(self,csv_path, height, width, transforms=None):
"""
:param csv_path:
:param height:
:param width:
:param transforms:
:param train: True is train data
"""
self.data = pd.read_csv(csv_path)
self.height = height
self.width = width
self.transforms = transforms
def __getitem__(self, index):
img_as_np = np.array(self.data.loc[index][0:]).reshape(self.height,self.width).astype("uint8")
if self.transforms is not None:
img_as_tensor = self.transforms(img_as_np)
return img_as_tensor
def __len__(self):
return len(self.data.index)
def get_data(path):
"""
:param path: train
:return: data_train, data_valid
"""
# labels
df = pd.DataFrame(pd.read_csv(path))
labels = df["label"]
labels_digit = []
for temp in labels:
labels_digit.append(temp)
labels_digit_numpy = np.array(labels_digit)
# data_train
temp_df = pd.DataFrame(pd.read_csv(path))
images = []
image = []
count = 0
for temp in temp_df.index:
print("count:", count)
line = temp_df.loc[temp].values[0:-1]
step = 28
image += [line[i:i+step] for i in range(0,784,step)]
# print("np.array(image).shape:", np.array(image).shape)
# np.savetxt("./temp_data.txt", image, fmt="%3d", newline="\n\n")
image = np.array(image)
images.append(image)
count += 1
# if count == 200:
# break
x_train = np.array(images)
x_train = x_train / 255.0
print("x_train.shape:", x_train.shape)
X_train, X_test, y_train, y_test = train_test_split(x_train,labels_digit_numpy,test_size=0.2, random_state=42)
print("X_train.shape:", X_train.shape)
print("y_train.shape:", y_train.shape)
print("X_test.shape:", X_test.shape)
print("y_test.shape:", y_test.shape)
"""
x_train.shape: (42000, 28, 28)
X_train.shape: (33600, 28, 28)
y_train.shape: (33600,)
X_test.shape: (8400, 28, 28)
y_test.shape: (8400,)
"""
return X_train, y_train, X_test, y_test
if __name__ == "__main__":
# path = "./data/train/train.csv"
# get_data(path)
transformations = transforms.Compose([transforms.ToTensor()])
train_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/train.csv", 28, 28, transformations,train=True)
valid_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/train.csv", 28, 28, transformations, train=False)
train_dataset_loader = torch.utils.data.DataLoader(dataset=train_custom_mnist_from_csv,
batch_size=10,
shuffle=False)
valid_dataset_loader = torch.utils.data.DataLoader(dataset=valid_custom_mnist_from_csv,
batch_size=10,
shuffle=False)
print("len(train_dataset_loader):", len(train_dataset_loader))
print("len(test_dataset_loader):", len(valid_dataset_loader))
data_process.py
import pandas as pd
head_split_train = 33600
tail_split_train = 8400
data_train = pd.read_csv("./data/train/train.csv").head(head_split_train)
data_train.to_csv("./data/train/data_train.csv", index=False)
data_valid = pd.read_csv("./data/train/train.csv").tail(tail_split_train)
data_valid.to_csv("./data/train/data_valid.csv",index=False)
model.py
import torch
import torch.nn as nn
import torch.nn.functional as F
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# 输入通道，输出10通道，kernel 5 * 5
self.conv1 = nn.Conv2d(in_channels=1, out_channels=10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, 5)
self.conv3 = nn.Conv2d(20, 40, 3)
self.mp = nn.MaxPool2d(2)
# fully connect
self.fc = nn.Linear(40, 10)# (in_features, out_features)
def forward(self, x):
# in_size = 64
in_size = x.size(0)# one batch 此时的x是包含batchsize维度为4的Tensor,即(batchsize,channels,x,y)
# x.size(0)值batchsize的值，把batchsize的值作为网络的in_size.
# x:64*1*28*28
x = F.relu(F.dropout(self.mp(self.conv1(x),0.25)))
# x:64*10*12*12 feature map = [(28-4)/2]^2=12*12
x = F.relu(F.dropout(self.mp(self.conv2(x))))
# x:64*20*4*4
x = F.relu(F.dropout(self.mp(self.conv3(x))))
x = x.view(in_size, -1)# flatten the tensor 相当于reshape
# x:64*20
x = self.fc(x)
return F.log_softmax(x) # 64 * 10
class CNNModel(nn.Module):
def __init__(self):
super(CNNModel, self).__init__()
# Convolution 1
self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, stride=1, padding=0)
self.relu1 = nn.ReLU()
#self.batchnormaliztion1 = nn.BatchNorm2d()
# Max pool1
self.maxpool1 = nn.MaxPool2d(kernel_size=2)
self.dropout1 = nn.Dropout2d(0.25)
# Convolution2
self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)
self.relu2 = nn.ReLU()
# Max pool2
self.maxpool2 = nn.MaxPool2d(kernel_size=2)
self.dropout2 = nn.Dropout2d(0.25)
# Fully connected1
self.fc1 = nn.Linear(32 * 4 * 4, 10)
def forward(self, x):
# Convolution 1
out = self.cnn1(x)
out = self.relu1(out)
# out = nn.BatchNorm2d(out)
# Max pool1
out = self.maxpool1(out)
out = self.dropout1(out)
# Convoltion2
out = self.cnn2(out)
out = self.relu2(out)
# Max pool2
out = self.maxpool2(out)
out = self.dropout2(out)
# flatten
out = out.view(out.size(0), -1)
# Linear function(readout)
out = self.fc1(out)
return out
train.py
import torch.nn as nn
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3"
import numpy as np
import pandas as pd
from torch.utils.data.dataset import Dataset
from torchvision import transforms
import torch
from data_loader import CustomDatasetFromCSV,Test_CustomDatasetFromCSV
from model import Net, CNNModel
import torch
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
# Training settings
batch_size = 10
# Data Loader (Input Pipeline)
transformations = transforms.Compose([transforms.ToTensor()])
train_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/data_train.csv", 28, 28, transformations)
#train_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/train.csv", 28, 28, transformations)
valid_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/data_valid.csv", 28, 28, transformations)
test_custom_mnist_from_csv = Test_CustomDatasetFromCSV("./data/test/test.csv", 28, 28, transformations)
train_dataset_loader = torch.utils.data.DataLoader(dataset=train_custom_mnist_from_csv,
batch_size=batch_size,
shuffle=False)
valid_dataset_loader = torch.utils.data.DataLoader(dataset=valid_custom_mnist_from_csv,
batch_size=batch_size,
shuffle=False)
test_dataset_loader = torch.utils.data.DataLoader(dataset=test_custom_mnist_from_csv,
batch_size=batch_size,
shuffle=False)
print("len(train_dataset_loader):", len(train_dataset_loader))
print("len(valid_dataset_loader):", len(valid_dataset_loader))
print("len(test_dataset_loader):", len(test_dataset_loader))
_model = CNNModel()
from torchsummary import summary
summary(_model.cuda(), input_size=(1, 28, 28))
optimizer = optim.SGD(_model.parameters(), lr=0.01, momentum=0.5)
def train(epoch):
for batch_idx, (data, target) in enumerate(train_dataset_loader):# batch_idx是enumerate()函数自带的索引，从0开始
# data.size():[-1, 1, 28, 28]
# target.size()：[64]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = data.to(device)
output = _model(data)
target = target.to(device)
# output:64*10(batchsize,10类别)
criteria = nn.CrossEntropyLoss()
loss = criteria(output, target)
if batch_idx % 200 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_dataset_loader.dataset),
100. * batch_idx / len(train_dataset_loader), loss.item()))
optimizer.zero_grad()# 所有参赛的梯度清零。
loss.backward()# 即反向传播求梯度。
optimizer.step()# 调用optimizer进行梯度下降更新参。
torch.save(_model,"./data/model.pt")
def _valid():
valid_loss = 0
correct = 0
for data, target in valid_dataset_loader:
data, target = Variable(data).cuda(), Variable(target).cuda()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = data.to(device)
target = target.to(device)
output = _model(data)
# print("output:", output)
# print("output.shape:", output.shape)# (batchsize,10类别)
# sum up batch loss
criteria = nn.CrossEntropyLoss()
# valid_loss += criteria(output, target, size_average=False).item()
valid_loss += criteria(output, target).item()
# get the index of the max log-probability
pred = output.data.max(1, keepdim=True)[1]
# print("---", pred)
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
valid_loss /= len(valid_dataset_loader.dataset)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
valid_loss, correct, len(valid_dataset_loader.dataset),
100. * correct / len(valid_dataset_loader.dataset)))
temp = 100. * correct / len(valid_dataset_loader.dataset)
print("temp:", temp.item())
return temp.item()
def Test_infer():
res = []
for data in test_dataset_loader:
data = Variable(data).cuda()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = data.to(device)
output = _model(data)
# get the index of the max log-probability
pred = output.data.max(1, keepdim=True)[1]
temp = pred.cuda().data.cpu().numpy().squeeze()
for i in range(len(temp)):
res.append(temp[i])
res_to_np = np.array(res)
import pandas as pd
pd = pd.read_csv("./data/sample_submission.csv")
print("res_to_np.shape:", res_to_np.shape)
pd["Label"] = res_to_np
pd.to_csv("./data/mnist_res.csv", index=False)
if __name__ == "__main__":
epochs = 2
for epoch in range(epochs):
train(epoch)
print("训练结束！")
max_score = 0
score = _valid()
if score > max_score:
Test_infer()
inference.py
import torch.nn as nn
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3"
import numpy as np
import pandas as pd
from torch.utils.data.dataset import Dataset
from torchvision import transforms
import torch
from data_loader import CustomDatasetFromCSV,Test_CustomDatasetFromCSV
from model import Net, CNNModel
import torch
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
# Training settings
batch_size = 10
# Data Loader (Input Pipeline)
transformations = transforms.Compose([transforms.ToTensor()])
#train_custom_mnist_from_csv = CustomDatasetFromCSV("./data/train/train.csv", 28, 28, transformations)
test_custom_mnist_from_csv = Test_CustomDatasetFromCSV("./data/test/test.csv", 28, 28, transformations)
test_dataset_loader = torch.utils.data.DataLoader(dataset=test_custom_mnist_from_csv,
batch_size=batch_size,
shuffle=False)
net2 = torch.load("./data/model.pt")
_model = net2
def Test_infer():
res = []
for data in test_dataset_loader:
data = Variable(data).cuda()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = data.to(device)
output = _model(data)
# get the index of the max log-probability
pred = output.data.max(1, keepdim=True)[1]
temp = pred.cuda().data.cpu().numpy().squeeze()
for i in range(len(temp)):
res.append(temp[i])
res_to_np = np.array(res)
import pandas as pd
pd = pd.read_csv("./data/sample_submission.csv")
print("res_to_np.shape:", res_to_np.shape)
pd["Label"] = res_to_np
pd.to_csv("./data/mnist_res.csv", index=False)
if __name__ == "__main__":
Test_infer()
