李宏毅_机器学习_作业1（详解）_COVID-19 Cases Prediction (Regression)
李宏毅_机器学习_作业1：COVID-19 Cases Prediction (Regression)
本文旨在，读懂hw1代码。李宏毅老师作业的完成有一个默认前提，掌握python基础语法,pytorch,numpy,pandas的使用。但并不是说必须将这些系统的学一遍才能完成作业，遇到不会的可直接baidu，我也是花了3天时间才读懂代码中各种函数，方法。作业使用jupyter notebook运行，pycharm亦可。
Download data
#gdown是一个Python的工具包，可下载Google drive的文件
#此处可以不通过gdown下载数据集，网上有线程的数据集，放到.ipynb同一个文件夹即可
!gdown --id '1kLSW_-cW2Huj7bh84YTdimGBOJaODiOS' --output covid.train.csv
!gdown --id '1iiI5qROrAhZn-o4FPqsE97bMzDEFvIdg' --output covid.test.csv
Import packages
# Numerical Operations
import math
import numpy as np
# Reading/Writing Data
import pandas as pd #pandas这里用来读文件
import os #os是python的标准库，对文件和文件夹进行操作
import csv
# For Progress Bar
from tqdm import tqdm #tqdm库用于生成训练时的进度条展示（需要pip）
# Pytorch
import torch
import torch.nn as nn #torch.nn是pytorch中自带的一个函数库，里面包含了神经网络中使用的一些常用函数
#建议学一下Dataset和DataLoader的使用
#Dataset:提供一种方式去获取数据及label，数据集在什么位置
#DataLoader:为之后的网络提供不同的数据形式
#random_split:无重复的随机划分数据集
from torch.utils.data import Dataset, DataLoader, random_split
# For plotting learning curve
#建议学一下tensorboard的使用，用于画loss曲线（需要pip）
from torch.utils.tensorboard import SummaryWriter
Some Utility Functions
def same_seed(seed):
'''Fixes random number generator seeds for reproducibility.'''
torch.backends.cudnn.deterministic = True #每次返回的卷积算法将是确定的
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available(): #为GPU设置种子
torch.cuda.manual_seed_all(seed)
#该方法在接下来DataLoader部分有调用，用于划分训练集和验证集
def train_valid_split(data_set, valid_ratio, seed):
'''Split provided training data into training set and validation set'''
#在DataLoader中调用时，valid_ratio是存储在字典中的值0.2
valid_set_size = int(valid_ratio * len(data_set)) #验证集的长度
train_set_size = len(data_set) - valid_set_size #训练集长度=data长度-验证集长度
#random_split无重复的随机划分数据集,第二个参数中传入的是需要划分成的两个数据集size
train_set, valid_set = random_split(data_set, [train_set_size, valid_set_size], generator=torch.Generator().manual_seed(seed))
#返回numpy数组形式
return np.array(train_set), np.array(valid_set)
def predict(test_loader, model, device):
model.eval() # Set your model to evaluation mode.
preds = []
for x in tqdm(test_loader):
x = x.to(device)
with torch.no_grad():
pred = model(x)
preds.append(pred.detach().cpu())
preds = torch.cat(preds, dim=0).numpy()
return preds
Dataset
这里涉及到python继承和构造函数__init__()的使用，建议先学习。
建议学习Dataset的使用
class COVID19Dataset(Dataset):
'''
x: Features.
y: Targets, if none, do prediction.
'''
#构造函数
def __init__(self, x, y=None): #y为缺省参数，如果不传参y则默认为none
if y is None:
self.y = y
else:
self.y = torch.FloatTensor(y)
self.x = torch.FloatTensor(x)
#获取具体的某一条数据
def __getitem__(self, idx):
if self.y is None:
return self.x[idx]
else:
return self.x[idx], self.y[idx]
#数据集长度
def __len__(self):
return len(self.x)
Neural Network Model
通过继承自nn.Module类来实现自定义模型
这里网络模型有4层：input(size=input_dim)->hidden(size=16)relu->hidden(size=8)relu->output
class My_Model(nn.Module):
def __init__(self, input_dim):
super(My_Model, self).__init__() #super()中的参数精准的指明了继承哪一个父类的方法
# TODO: modify model's structure, be aware of dimensions.
#nn是pytorch中自带的一个函数库，里面包含了神经网络中使用的一些常用函数
self.layers = nn.Sequential(
nn.Linear(input_dim, 16),#设置网络中的全连接层
nn.ReLU(),
nn.Linear(16, 8),
nn.ReLU(),
nn.Linear(8, 1)
)
#定义前向传播
def forward(self, x):
x = self.layers(x)
x = x.squeeze(1) # (B, 1) -> (B)
return x
Feature Selection
选择训练时需要使用的特征
Choose features you deem useful by modifying the function below.
def select_feat(train_data, valid_data, test_data, select_all=True): #valid_data验证集用于防止过拟合
'''Selects useful features to perform regression'''
#[:,-1]冒号表示第一个维度选择全部，即全部行，-1：表示倒数第一个元素，即label
y_train, y_valid = train_data[:,-1], valid_data[:,-1] #这里是数组的切片方式,建议学
#[:,:-1]第一个冒号表示第一个维度选择全部，即全部行，:-1表示在第二个维度上从开始至最后一个元素（不包含最后一个元素）即选择了除最后一列的所有特征
raw_x_train, raw_x_valid, raw_x_test = train_data[:,:-1], valid_data[:,:-1], test_data
if select_all:
#选择所有特征
feat_idx = list(range(raw_x_train.shape[1])) #raw_x_train形状2160*95 raw_x_train.shape[1]表示第二个维度元素的个数95
else:#自定义特征
feat_idx = [0,1,2,3,4] # TODO: Select suitable feature columns.
return raw_x_train[:,feat_idx], raw_x_valid[:,feat_idx], raw_x_test[:,feat_idx], y_train, y_valid
Training Loop
训练的过程
def trainer(train_loader, valid_loader, model, config, device):
criterion = nn.MSELoss(reduction='mean') # Define your loss function, do not modify this.
# Define your optimization algorithm.
# TODO: Please check https://pytorch.org/docs/stable/optim.html to get more available algorithms.
# TODO: L2 regularization (optimizer(weight decay...) or implement by your self).
#优化算法SGD
optimizer = torch.optim.SGD(model.parameters(), lr=config['learning_rate'], momentum=0.9)
#带动量的随机梯度下降Momentum-SGD
writer = SummaryWriter() # Writer of tensoboard.
if not os.path.isdir('./models'): #判断models文件夹是否存在
os.mkdir('./models') # Create directory of saving models.
n_epochs, best_loss, step, early_stop_count = config['n_epochs'], math.inf, 0, 0
for epoch in range(n_epochs): #3000轮
#开启训练模式batchNorm层，dropout层等用于优化训练而添加的网络层开启
model.train() # Set your model to train mode.
#记录loss函数值
loss_record = []
# tqdm is a package to visualize your training progress.
train_pbar = tqdm(train_loader, position=0, leave=True)
#tqdm用在dataloader上其实是对每个batch和batch总数做的进度条
for x, y in train_pbar:
optimizer.zero_grad() # Set gradient to zero.
# x.to(device)将所有最开始读取数据时的tensor变量copy一份到device所指定的GPU上去，
#之后的运算都在GPU上进行。
x, y = x.to(device), y.to(device) # Move your data to device.
pred = model(x) #prediction 前向传播 与model.forward(x)含义相同
loss = criterion(pred, y) #loss function
loss.backward() # Compute gradient(backpropagation)计算梯度
optimizer.step() # Update parameters.
step += 1
loss_record.append(loss.detach().item())
# Display current epoch number and loss on tqdm progress bar.
train_pbar.set_description(f'Epoch [{epoch+1}/{n_epochs}]')
train_pbar.set_postfix({'loss': loss.detach().item()})
mean_train_loss = sum(loss_record)/len(loss_record) #平均loss
writer.add_scalar('Loss/train', mean_train_loss, step) #tensoboard画出loss曲线
model.eval() # Set your model to evaluation mode.在评估模式下，batchNorm层，dropout层等用于优化训练而添加的网络层会被关闭
#从狭义来讲，验证集没有参与梯度下降的过程，也就是说是没有经过训练的；
#但从广义上来看，验证集却参与了一个“人工调参”的过程，我们根据验证集的结果调节了迭代数、调节了学习率等等，使得结果在验证集上最优
loss_record = []
for x, y in valid_loader:
x, y = x.to(device), y.to(device)
with torch.no_grad():
pred = model(x)
loss = criterion(pred, y)
#少了计算梯度，更新参数的过程
loss_record.append(loss.item())
mean_valid_loss = sum(loss_record)/len(loss_record)
print(f'Epoch [{epoch+1}/{n_epochs}]: Train loss: {mean_train_loss:.4f}, Valid loss: {mean_valid_loss:.4f}')
writer.add_scalar('Loss/valid', mean_valid_loss, step)
if mean_valid_loss < best_loss: #验证集上的loss小于best_loss
best_loss = mean_valid_loss #更新best_loss,保存模型
torch.save(model.state_dict(), config['save_path']) # Save your best model
print('Saving model with loss {:.3f}...'.format(best_loss))
early_stop_count = 0
else: #验证集上的loss并没有更好
early_stop_count += 1
#验证集上的loss超过400轮没有小于best_loss
if early_stop_count >= config['early_stop']:
print('\nModel is not improving, so we halt the training session.')
return
Configurations
各种参数
device = 'cuda' if torch.cuda.is_available() else 'cpu'
config = { #字典设置参数
'seed': 5201314, # Your seed number, you can pick your lucky number. :)
'select_all': True, # Whether to use all features.
'valid_ratio': 0.2, # validation_size = train_size * valid_ratio
'n_epochs': 3000, # Number of epochs.
'batch_size': 256,
'learning_rate': 1e-5,
'early_stop': 400, # If model has not improved for this many consecutive epochs, stop training.
'save_path': './models/model.ckpt' # Your model will be saved here.
}
Dataloader
# Set seed for reproducibility
same_seed(config['seed'])
# train_data size: 2699 x 118 (id + 37 states + 16 features x 5 days)
# test_data size: 1078 x 117 (without last day's positive rate)
train_data, test_data = pd.read_csv('./covid.train.csv').values, pd.read_csv('./covid.test.csv').values
#train_valid_split是上面定义的函数，无覆盖的随机划分数据集
train_data, valid_data = train_valid_split(train_data, config['valid_ratio'], config['seed'])
# Print out the data size.
print(f"""train_data size: {train_data.shape}
valid_data size: {valid_data.shape}
test_data size: {test_data.shape}""")
# Select features
x_train, x_valid, x_test, y_train, y_valid = select_feat(train_data, valid_data, test_data, config['select_all'])
# Print out the number of features.
print(f'number of features: {x_train.shape[1]}')
#生成dataset实例
train_dataset, valid_dataset, test_dataset = COVID19Dataset(x_train, y_train), \
COVID19Dataset(x_valid, y_valid), \
COVID19Dataset(x_test)
# Pytorch data loader loads pytorch dataset into batches.256
train_loader = DataLoader(train_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
#pin_memory默认false，打开后更快
valid_loader = DataLoader(valid_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
test_loader = DataLoader(test_dataset, batch_size=config['batch_size'], shuffle=False, pin_memory=True)
Start training!
model = My_Model(input_dim=x_train.shape[1]).to(device) # put your model and data on the same computation device.
trainer(train_loader, valid_loader, model, config, device)
Plot learning curves with tensorboard (optional)
可视化训练过程
If this block does not display your learning curve, please wait for few minutes, and re-run this block. It might take some time to load your logging information.
未加载出曲线，等几分钟重新运行
%reload_ext tensorboard
%tensorboard --logdir=./runs/
Testing
测试结果保存在pred.csv中
def save_pred(preds, file):
''' Save predictions to specified file '''
with open(file, 'w') as fp:
writer = csv.writer(fp)
writer.writerow(['id', 'tested_positive'])
for i, p in enumerate(preds):
writer.writerow([i, p])
model = My_Model(input_dim=x_train.shape[1]).to(device)
model.load_state_dict(torch.load(config['save_path']))
preds = predict(test_loader, model, device)
save_pred(preds, 'pred.csv')