动手学深度学习之kaggle竞赛
实战Kaggle比赛：图像分类(CIFAR-10)
import collections
import math
import os
import shutil
import pandas as pd
import torch
import torchvision
from torch import nn
from d2l import torch as d2l
我们提供包含前1000个训练图像和5个随机测试图像的数据集的小规模样本
d2l.DATA_HUB['cifar10_tiny'] = (d2l.DATA_URL + 'kaggle_cifar10_tiny.zip',
'2068874e4b9a9f0fb07ebe0ad2b29754449ccacd')
demo = True
if demo:
data_dir = d2l.download_extract('cifar10_tiny')
else:
data_dir = '../data/cifar-10/'
Downloading ../data/kaggle_cifar10_tiny.zip from http://d2l-data.s3-accelerate.amazonaws.com/kaggle_cifar10_tiny.zip...
整理数据集
def read_csv_labels(fname):
"""读取‘fname’来给标签字典返回一个文件名"""
with open(fname, 'r') as f:
lines = f.readlines()[1:]
tokens = [l.rstrip().split(',') for l in lines]
return dict(((name, label) for name, label in tokens))
labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))
labels
{'1': 'frog',
'2': 'truck',
'3': 'truck',
'4': 'deer',
'5': 'automobile',
'6': 'automobile',
'7': 'bird',
'8': 'horse',
'9': 'ship',
'10': 'cat',
'11': 'deer',
'12': 'horse',
'13': 'horse',
'14': 'bird',
'15': 'truck',
'16': 'truck',
'17': 'truck',
'18': 'cat',
'19': 'bird',
'20': 'frog',
'21': 'deer',
'22': 'cat',
'23': 'frog',
'24': 'frog',
'25': 'bird',
'26': 'frog',
'27': 'cat',
'28': 'dog',
'29': 'deer',
'30': 'airplane',
'31': 'airplane',
'32': 'truck',
'33': 'automobile',
'34': 'cat',
'35': 'deer',
'36': 'airplane',
'37': 'cat',
'38': 'horse',
'39': 'cat',
'40': 'cat',
'41': 'dog',
'42': 'bird',
'43': 'bird',
'44': 'horse',
'45': 'automobile',
'46': 'automobile',
'47': 'automobile',
'48': 'bird',
'49': 'bird',
'50': 'airplane',
'51': 'truck',
'52': 'dog',
'53': 'horse',
'54': 'truck',
'55': 'bird',
'56': 'bird',
'57': 'dog',
'58': 'bird',
'59': 'deer',
'60': 'cat',
'61': 'automobile',
'62': 'automobile',
'63': 'ship',
'64': 'bird',
'65': 'automobile',
'66': 'automobile',
'67': 'deer',
'68': 'truck',
'69': 'horse',
'70': 'ship',
'71': 'dog',
'72': 'truck',
'73': 'frog',
'74': 'horse',
'75': 'cat',
'76': 'automobile',
'77': 'truck',
'78': 'airplane',
'79': 'cat',
'80': 'automobile',
'81': 'cat',
'82': 'dog',
'83': 'deer',
'84': 'dog',
'85': 'horse',
'86': 'horse',
'87': 'deer',
'88': 'horse',
'89': 'truck',
'90': 'deer',
'91': 'bird',
'92': 'cat',
'93': 'ship',
'94': 'airplane',
'95': 'automobile',
'96': 'frog',
'97': 'automobile',
'98': 'automobile',
'99': 'deer',
'100': 'automobile',
'101': 'ship',
'102': 'cat',
'103': 'truck',
'104': 'frog',
'105': 'frog',
'106': 'automobile',
'107': 'ship',
'108': 'dog',
'109': 'bird',
'110': 'truck',
'111': 'truck',
'112': 'ship',
'113': 'automobile',
'114': 'horse',
'115': 'horse',
'116': 'airplane',
'117': 'airplane',
'118': 'frog',
'119': 'truck',
'120': 'automobile',
'121': 'bird',
'122': 'bird',
'123': 'truck',
'124': 'bird',
'125': 'frog',
'126': 'frog',
'127': 'automobile',
'128': 'truck',
'129': 'dog',
'130': 'airplane',
'131': 'deer',
'132': 'horse',
'133': 'frog',
'134': 'horse',
'135': 'automobile',
'136': 'ship',
'137': 'automobile',
'138': 'automobile',
'139': 'bird',
'140': 'ship',
'141': 'automobile',
'142': 'cat',
'143': 'cat',
'144': 'frog',
'145': 'bird',
'146': 'deer',
'147': 'truck',
'148': 'truck',
'149': 'dog',
'150': 'deer',
'151': 'cat',
'152': 'frog',
'153': 'horse',
'154': 'deer',
'155': 'frog',
'156': 'ship',
'157': 'dog',
'158': 'dog',
'159': 'deer',
'160': 'cat',
'161': 'automobile',
'162': 'ship',
'163': 'deer',
'164': 'horse',
'165': 'frog',
'166': 'airplane',
'167': 'truck',
'168': 'dog',
'169': 'automobile',
'170': 'cat',
'171': 'ship',
'172': 'bird',
'173': 'horse',
'174': 'dog',
'175': 'cat',
'176': 'deer',
'177': 'automobile',
'178': 'dog',
'179': 'horse',
'180': 'airplane',
'181': 'deer',
'182': 'horse',
'183': 'dog',
'184': 'dog',
'185': 'automobile',
'186': 'airplane',
'187': 'truck',
'188': 'frog',
'189': 'truck',
'190': 'airplane',
'191': 'ship',
'192': 'horse',
'193': 'ship',
'194': 'ship',
'195': 'bird',
'196': 'dog',
'197': 'bird',
'198': 'cat',
'199': 'dog',
'200': 'airplane',
'201': 'frog',
'202': 'automobile',
'203': 'truck',
'204': 'cat',
'205': 'frog',
'206': 'truck',
'207': 'automobile',
'208': 'cat',
'209': 'truck',
'210': 'frog',
'211': 'frog',
'212': 'horse',
'213': 'automobile',
'214': 'airplane',
'215': 'truck',
'216': 'dog',
'217': 'ship',
'218': 'dog',
'219': 'bird',
'220': 'truck',
'221': 'airplane',
'222': 'ship',
'223': 'ship',
'224': 'airplane',
'225': 'frog',
'226': 'truck',
'227': 'automobile',
'228': 'automobile',
'229': 'frog',
'230': 'cat',
'231': 'horse',
'232': 'frog',
'233': 'frog',
'234': 'airplane',
'235': 'frog',
'236': 'frog',
'237': 'automobile',
'238': 'horse',
'239': 'automobile',
'240': 'dog',
'241': 'ship',
'242': 'cat',
'243': 'frog',
'244': 'frog',
'245': 'ship',
'246': 'frog',
'247': 'ship',
'248': 'deer',
'249': 'frog',
'250': 'frog',
'251': 'automobile',
'252': 'cat',
'253': 'ship',
'254': 'cat',
'255': 'deer',
'256': 'automobile',
'257': 'horse',
'258': 'automobile',
'259': 'cat',
'260': 'ship',
'261': 'dog',
'262': 'automobile',
'263': 'automobile',
'264': 'deer',
'265': 'airplane',
'266': 'truck',
'267': 'cat',
'268': 'horse',
'269': 'deer',
'270': 'truck',
'271': 'truck',
'272': 'bird',
'273': 'deer',
'274': 'truck',
'275': 'truck',
'276': 'automobile',
'277': 'airplane',
'278': 'dog',
'279': 'truck',
'280': 'airplane',
'281': 'ship',
'282': 'bird',
'283': 'automobile',
'284': 'bird',
'285': 'airplane',
'286': 'dog',
'287': 'frog',
'288': 'cat',
'289': 'bird',
'290': 'horse',
'291': 'ship',
'292': 'ship',
'293': 'frog',
'294': 'airplane',
'295': 'horse',
'296': 'truck',
'297': 'deer',
'298': 'dog',
'299': 'frog',
'300': 'deer',
'301': 'bird',
'302': 'automobile',
'303': 'automobile',
'304': 'bird',
'305': 'automobile',
'306': 'dog',
'307': 'truck',
'308': 'truck',
'309': 'airplane',
'310': 'ship',
'311': 'deer',
'312': 'automobile',
'313': 'automobile',
'314': 'frog',
'315': 'cat',
'316': 'cat',
'317': 'truck',
'318': 'airplane',
'319': 'horse',
'320': 'truck',
'321': 'horse',
'322': 'horse',
'323': 'truck',
'324': 'automobile',
'325': 'dog',
'326': 'automobile',
'327': 'frog',
'328': 'frog',
'329': 'ship',
'330': 'horse',
'331': 'automobile',
'332': 'cat',
'333': 'airplane',
'334': 'cat',
'335': 'cat',
'336': 'bird',
'337': 'deer',
'338': 'dog',
'339': 'horse',
'340': 'dog',
'341': 'truck',
'342': 'airplane',
'343': 'cat',
'344': 'deer',
'345': 'airplane',
'346': 'deer',
'347': 'deer',
'348': 'frog',
'349': 'airplane',
'350': 'airplane',
'351': 'frog',
'352': 'frog',
'353': 'airplane',
'354': 'ship',
'355': 'automobile',
'356': 'frog',
'357': 'bird',
'358': 'truck',
'359': 'bird',
'360': 'dog',
'361': 'truck',
'362': 'frog',
'363': 'horse',
'364': 'deer',
'365': 'automobile',
'366': 'ship',
'367': 'horse',
'368': 'cat',
'369': 'frog',
'370': 'truck',
'371': 'cat',
'372': 'airplane',
'373': 'deer',
'374': 'airplane',
'375': 'dog',
'376': 'automobile',
'377': 'airplane',
'378': 'cat',
'379': 'deer',
'380': 'ship',
'381': 'dog',
'382': 'deer',
'383': 'horse',
'384': 'bird',
'385': 'cat',
'386': 'truck',
'387': 'horse',
'388': 'frog',
'389': 'horse',
'390': 'automobile',
'391': 'deer',
'392': 'horse',
'393': 'airplane',
'394': 'automobile',
'395': 'horse',
'396': 'cat',
'397': 'automobile',
'398': 'ship',
'399': 'deer',
'400': 'deer',
'401': 'bird',
'402': 'airplane',
'403': 'bird',
'404': 'bird',
'405': 'airplane',
'406': 'airplane',
'407': 'truck',
'408': 'airplane',
'409': 'truck',
'410': 'frog',
'411': 'ship',
'412': 'bird',
'413': 'horse',
'414': 'horse',
'415': 'deer',
'416': 'airplane',
'417': 'cat',
'418': 'airplane',
'419': 'ship',
'420': 'truck',
'421': 'deer',
'422': 'bird',
'423': 'horse',
'424': 'bird',
'425': 'dog',
'426': 'bird',
'427': 'dog',
'428': 'automobile',
'429': 'truck',
'430': 'deer',
'431': 'ship',
'432': 'dog',
'433': 'automobile',
'434': 'horse',
'435': 'deer',
'436': 'deer',
'437': 'airplane',
'438': 'frog',
'439': 'truck',
'440': 'airplane',
'441': 'horse',
'442': 'ship',
'443': 'ship',
'444': 'truck',
'445': 'truck',
'446': 'cat',
'447': 'cat',
'448': 'deer',
'449': 'airplane',
'450': 'deer',
'451': 'dog',
'452': 'frog',
'453': 'frog',
'454': 'airplane',
'455': 'automobile',
'456': 'airplane',
'457': 'ship',
'458': 'airplane',
'459': 'deer',
'460': 'ship',
'461': 'ship',
'462': 'automobile',
'463': 'dog',
'464': 'bird',
'465': 'frog',
'466': 'ship',
'467': 'automobile',
'468': 'airplane',
'469': 'airplane',
'470': 'horse',
'471': 'horse',
'472': 'dog',
'473': 'truck',
'474': 'frog',
'475': 'bird',
'476': 'ship',
'477': 'cat',
'478': 'deer',
'479': 'horse',
'480': 'cat',
'481': 'truck',
'482': 'airplane',
'483': 'automobile',
'484': 'bird',
'485': 'deer',
'486': 'ship',
'487': 'automobile',
'488': 'ship',
'489': 'frog',
'490': 'deer',
'491': 'deer',
'492': 'dog',
'493': 'horse',
'494': 'automobile',
'495': 'cat',
'496': 'truck',
'497': 'ship',
'498': 'airplane',
'499': 'automobile',
'500': 'horse',
'501': 'dog',
'502': 'ship',
'503': 'bird',
'504': 'ship',
'505': 'airplane',
'506': 'deer',
'507': 'automobile',
'508': 'ship',
'509': 'truck',
'510': 'ship',
'511': 'bird',
'512': 'truck',
'513': 'truck',
'514': 'bird',
'515': 'horse',
'516': 'dog',
'517': 'horse',
'518': 'cat',
'519': 'ship',
'520': 'ship',
'521': 'deer',
'522': 'deer',
'523': 'bird',
'524': 'horse',
'525': 'automobile',
'526': 'frog',
'527': 'deer',
'528': 'airplane',
'529': 'deer',
'530': 'frog',
'531': 'truck',
'532': 'horse',
'533': 'frog',
'534': 'bird',
'535': 'dog',
'536': 'dog',
'537': 'automobile',
'538': 'horse',
'539': 'bird',
'540': 'bird',
'541': 'bird',
'542': 'truck',
'543': 'dog',
'544': 'deer',
'545': 'bird',
'546': 'horse',
'547': 'ship',
'548': 'automobile',
'549': 'cat',
'550': 'deer',
'551': 'cat',
'552': 'horse',
'553': 'frog',
'554': 'truck',
'555': 'ship',
'556': 'airplane',
'557': 'frog',
'558': 'airplane',
'559': 'bird',
'560': 'bird',
'561': 'bird',
'562': 'automobile',
'563': 'ship',
'564': 'deer',
'565': 'airplane',
'566': 'automobile',
'567': 'ship',
'568': 'ship',
'569': 'automobile',
'570': 'dog',
'571': 'horse',
'572': 'frog',
'573': 'deer',
'574': 'dog',
'575': 'ship',
'576': 'horse',
'577': 'automobile',
'578': 'truck',
'579': 'automobile',
'580': 'truck',
'581': 'ship',
'582': 'deer',
'583': 'horse',
'584': 'cat',
'585': 'ship',
'586': 'ship',
'587': 'bird',
'588': 'frog',
'589': 'frog',
'590': 'horse',
'591': 'automobile',
'592': 'frog',
'593': 'ship',
'594': 'automobile',
'595': 'truck',
'596': 'horse',
'597': 'ship',
'598': 'cat',
'599': 'airplane',
'600': 'automobile',
'601': 'airplane',
'602': 'ship',
'603': 'ship',
'604': 'cat',
'605': 'airplane',
'606': 'airplane',
'607': 'automobile',
'608': 'dog',
'609': 'airplane',
'610': 'ship',
'611': 'ship',
'612': 'horse',
'613': 'truck',
'614': 'truck',
'615': 'airplane',
'616': 'truck',
'617': 'deer',
'618': 'automobile',
'619': 'cat',
'620': 'frog',
'621': 'frog',
'622': 'deer',
'623': 'deer',
'624': 'horse',
'625': 'dog',
'626': 'frog',
'627': 'airplane',
'628': 'ship',
'629': 'airplane',
'630': 'cat',
'631': 'bird',
'632': 'ship',
'633': 'deer',
'634': 'frog',
'635': 'truck',
'636': 'truck',
'637': 'horse',
'638': 'airplane',
'639': 'cat',
'640': 'cat',
'641': 'frog',
'642': 'horse',
'643': 'deer',
'644': 'truck',
'645': 'automobile',
'646': 'frog',
'647': 'bird',
'648': 'horse',
'649': 'bird',
'650': 'bird',
'651': 'airplane',
'652': 'frog',
'653': 'horse',
'654': 'dog',
'655': 'horse',
'656': 'frog',
'657': 'ship',
'658': 'truck',
'659': 'airplane',
'660': 'truck',
'661': 'deer',
'662': 'deer',
'663': 'horse',
'664': 'airplane',
'665': 'truck',
'666': 'deer',
'667': 'truck',
'668': 'frog',
'669': 'truck',
'670': 'deer',
'671': 'dog',
'672': 'horse',
'673': 'truck',
'674': 'bird',
'675': 'deer',
'676': 'dog',
'677': 'automobile',
'678': 'deer',
'679': 'cat',
'680': 'truck',
'681': 'frog',
'682': 'dog',
'683': 'frog',
'684': 'truck',
'685': 'cat',
'686': 'cat',
'687': 'dog',
'688': 'airplane',
'689': 'horse',
'690': 'bird',
'691': 'automobile',
'692': 'cat',
'693': 'frog',
'694': 'deer',
'695': 'airplane',
'696': 'airplane',
'697': 'bird',
'698': 'dog',
'699': 'airplane',
'700': 'automobile',
'701': 'airplane',
'702': 'bird',
'703': 'cat',
'704': 'truck',
'705': 'ship',
'706': 'deer',
'707': 'truck',
'708': 'ship',
'709': 'airplane',
'710': 'bird',
'711': 'frog',
'712': 'deer',
'713': 'deer',
'714': 'airplane',
'715': 'automobile',
'716': 'ship',
'717': 'ship',
'718': 'cat',
'719': 'frog',
'720': 'truck',
'721': 'frog',
'722': 'frog',
'723': 'horse',
'724': 'ship',
'725': 'bird',
'726': 'deer',
'727': 'dog',
'728': 'horse',
'729': 'frog',
'730': 'dog',
'731': 'cat',
'732': 'airplane',
'733': 'dog',
'734': 'airplane',
'735': 'dog',
'736': 'airplane',
'737': 'ship',
'738': 'bird',
'739': 'frog',
'740': 'horse',
'741': 'cat',
'742': 'ship',
'743': 'bird',
'744': 'automobile',
'745': 'horse',
'746': 'frog',
'747': 'horse',
'748': 'automobile',
'749': 'airplane',
'750': 'truck',
'751': 'dog',
'752': 'dog',
'753': 'airplane',
'754': 'automobile',
'755': 'horse',
'756': 'frog',
'757': 'truck',
'758': 'airplane',
'759': 'deer',
'760': 'horse',
'761': 'horse',
'762': 'automobile',
'763': 'dog',
'764': 'truck',
'765': 'deer',
'766': 'airplane',
'767': 'ship',
'768': 'dog',
'769': 'truck',
'770': 'truck',
'771': 'frog',
'772': 'horse',
'773': 'automobile',
'774': 'ship',
'775': 'cat',
'776': 'bird',
'777': 'cat',
'778': 'ship',
'779': 'bird',
'780': 'bird',
'781': 'deer',
'782': 'frog',
'783': 'airplane',
'784': 'airplane',
'785': 'dog',
'786': 'cat',
'787': 'ship',
'788': 'bird',
'789': 'cat',
'790': 'horse',
'791': 'bird',
'792': 'truck',
'793': 'cat',
'794': 'ship',
'795': 'horse',
'796': 'ship',
'797': 'bird',
'798': 'horse',
'799': 'truck',
'800': 'airplane',
'801': 'bird',
'802': 'cat',
'803': 'bird',
'804': 'bird',
'805': 'bird',
'806': 'cat',
'807': 'cat',
'808': 'frog',
'809': 'bird',
'810': 'cat',
'811': 'bird',
'812': 'ship',
'813': 'airplane',
'814': 'dog',
'815': 'dog',
'816': 'automobile',
'817': 'deer',
'818': 'dog',
'819': 'frog',
'820': 'frog',
'821': 'bird',
'822': 'horse',
'823': 'airplane',
'824': 'automobile',
'825': 'horse',
'826': 'horse',
'827': 'ship',
'828': 'bird',
'829': 'truck',
'830': 'bird',
'831': 'bird',
'832': 'deer',
'833': 'bird',
'834': 'automobile',
'835': 'automobile',
'836': 'automobile',
'837': 'frog',
'838': 'frog',
'839': 'frog',
'840': 'dog',
'841': 'automobile',
'842': 'automobile',
'843': 'horse',
'844': 'airplane',
'845': 'deer',
'846': 'cat',
'847': 'cat',
'848': 'horse',
'849': 'automobile',
'850': 'bird',
'851': 'cat',
'852': 'dog',
'853': 'dog',
'854': 'dog',
'855': 'frog',
'856': 'automobile',
'857': 'deer',
'858': 'cat',
'859': 'horse',
'860': 'ship',
'861': 'ship',
'862': 'cat',
'863': 'frog',
'864': 'frog',
'865': 'bird',
'866': 'cat',
'867': 'airplane',
'868': 'truck',
'869': 'deer',
'870': 'cat',
'871': 'ship',
'872': 'airplane',
'873': 'airplane',
'874': 'automobile',
'875': 'automobile',
'876': 'dog',
'877': 'deer',
'878': 'truck',
'879': 'cat',
'880': 'automobile',
'881': 'ship',
'882': 'truck',
'883': 'cat',
'884': 'truck',
'885': 'truck',
'886': 'bird',
'887': 'truck',
'888': 'deer',
'889': 'ship',
'890': 'bird',
'891': 'truck',
'892': 'ship',
'893': 'ship',
'894': 'automobile',
'895': 'dog',
'896': 'cat',
'897': 'frog',
'898': 'ship',
'899': 'horse',
'900': 'frog',
'901': 'truck',
'902': 'ship',
'903': 'airplane',
'904': 'frog',
'905': 'deer',
'906': 'airplane',
'907': 'airplane',
'908': 'bird',
'909': 'dog',
'910': 'ship',
'911': 'bird',
'912': 'airplane',
'913': 'bird',
'914': 'horse',
'915': 'frog',
'916': 'truck',
'917': 'horse',
'918': 'automobile',
'919': 'dog',
'920': 'dog',
'921': 'frog',
'922': 'frog',
'923': 'cat',
'924': 'frog',
'925': 'bird',
'926': 'deer',
'927': 'horse',
'928': 'airplane',
'929': 'dog',
'930': 'frog',
'931': 'deer',
'932': 'frog',
'933': 'dog',
'934': 'bird',
'935': 'deer',
'936': 'frog',
'937': 'automobile',
'938': 'frog',
'939': 'airplane',
'940': 'deer',
'941': 'airplane',
'942': 'cat',
'943': 'automobile',
'944': 'ship',
'945': 'dog',
'946': 'deer',
'947': 'deer',
'948': 'automobile',
'949': 'horse',
'950': 'cat',
'951': 'truck',
'952': 'deer',
'953': 'horse',
'954': 'truck',
'955': 'horse',
'956': 'cat',
'957': 'horse',
'958': 'bird',
'959': 'ship',
'960': 'deer',
'961': 'frog',
'962': 'frog',
'963': 'automobile',
'964': 'bird',
'965': 'truck',
'966': 'airplane',
'967': 'deer',
'968': 'ship',
'969': 'horse',
'970': 'cat',
'971': 'truck',
'972': 'ship',
'973': 'horse',
'974': 'horse',
'975': 'airplane',
'976': 'bird',
'977': 'deer',
'978': 'automobile',
'979': 'automobile',
'980': 'deer',
'981': 'automobile',
'982': 'dog',
'983': 'deer',
'984': 'airplane',
'985': 'dog',
'986': 'frog',
'987': 'bird',
'988': 'ship',
'989': 'dog',
'990': 'airplane',
'991': 'bird',
'992': 'automobile',
'993': 'cat',
'994': 'dog',
'995': 'horse',
'996': 'cat',
'997': 'dog',
'998': 'automobile',
'999': 'cat',
'1000': 'dog'}
将验证集从原始的训练集中拆分出来
# 在pytorch中有一个比较简单但很常用的加载数据的方式就是先将文件夹创建好，然后文件夹名字为label，然后将这个label的训练数据放进去
# 这个函数的作用就是创建子文件夹，然后将图片搬过去
def copyfile(filename, target_dir):
"""文件复制到目标目录"""
os.makedirs(target_dir, exist_ok=True)
shutil.copy(filename, target_dir)
# 根目录：train_valid_test。下面有train文件夹，包含训练数据。valid包含验证数据，train_valid原始的train文件夹
def reorg_train_valid(data_dir, labels, valid_ratio):
n = collections.Counter(labels.values()).most_common()[-1][1]
n_valid_per_label = max(1, math.floor(n * valid_ratio))
label_count = {}
for train_file in os.listdir(os.path.join(data_dir, 'train')):
label = labels[train_file.split('.')[0]]
fname = os.path.join(data_dir, 'train', train_file)
copyfile(
fname,
os.path.join(data_dir, 'train_valid_set', 'train_valid', label))
if label not in label_count or label_count[label] < n_valid_per_label:
copyfile(
fname,
os.path.join(data_dir, 'train_valid_test', 'valid', label))
else:
copyfile(
fname,
os.path.join(data_dir, 'train_valid_test', 'train', label))
return n_valid_per_label
在预测期间整理测试集，以方便读取
def reorg_test(data_dir):
for test_file in os.listdir(os.path.join(data_dir, 'test')):
copyfile(
os.path.join(data_dir, 'test', test_file),
os.path.join(data_dir, 'train_valid_test', 'test', 'unknown'))
调用前面定义的函数
def reorg_cifar10_data(data_dir, valid_ratio):
labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))
reorg_train_valid(data_dir, labels, valid_ratio)
reorg_test(data_dir)
batch_size = 32 if demo else 128
valid_ratio = 0.1
reorg_cifar10_data(data_dir, valid_ratio)
图像增广
transform_train = torchvision.transforms.Compose([
torchvision.transforms.Resize(40), # 将图片放大到40*40
torchvision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0),
ratio=(1.0, 1.0)), # 随机裁剪
torchvision.transforms.RandomHorizontalFlip(), # 水平调整
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])]) # 对RGB三个channel
transform_test = torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])])
读取由原始图像组成的数据集
train_ds, train_valid_ds = [
torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_train) for folder in ['train', 'train_valid']]
valid_ds, test_ds = [
torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_test) for folder in ['valid', 'test']]
指定上面定义的所有图像增广操作
train_iter, train_valid_iter = [
torch.utils.data.DataLoader(dataset, batch_size, shuffle=True,
drop_last=True) # drop_last表示如果最后一个批量大小不够的话，就丢掉
for dataset in (train_ds, train_valid_ds)]
valid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,
drop_last=True)
test_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,
drop_last=False)
模型
def get_net():
num_classes = 10
net = d2l.resnet18(num_classes, 3) # 3的意思就是RGB三通道
return net
loss = nn.CrossEntropyLoss(reduction="none") # reduction=‘none’表示不要加起来
训练函数
def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay): # lr_period,lr_decay，这里的意思就是每隔几次迭代学习率降低
trainer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9,
weight_decay=wd)
scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay) # 这个函数的作用就是每个多少个迭代，将学习率乘以lr_decay
num_batches, timer = len(train_iter), d2l.Timer()
legend = ['train loss', 'train acc']
if valid_iter is not None:
legend.append('valid acc')
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
legend=legend)
net = nn.DataParallel(net, device_ids=devices).to(devices[0])
for epoch in range(num_epochs):
net.train()
metric = d2l.Accumulator(3)
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = d2l.train_batch_ch13(net, features, labels, loss,
trainer, devices)
metric.add(l, acc, labels.shape[0])
timer.stop()
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(
epoch + (i + 1) / num_batches,
(metric[0] / metric[2], metric[1] / metric[2], None))
if valid_iter is not None:
valid_acc = d2l.evaluate_accuracy_gpu(net, valid_iter)
animator.add(epoch + 1, (None, None, valid_acc))
scheduler.step() # 这里scheduler的原因就是之前已经将trainer给了schdeuler
measures = (f'train loss {metric[0] / metric[2]:.3f}, '
f'train acc {metric[1] / metric[2]:.3f}')
if valid_iter is not None:
measures += f', valid acc {valid_acc:.3f}'
print(measures + f'\n{metric[2] * num_epochs / timer.sum():.1f}'
f' examples/sec on {str(devices)}')
训练和验证模型
devices, num_epochs, lr, wd = d2l.try_all_gpus(), 20, 2e-4, 5e-4
lr_period, lr_decay, net = 4, 0.9, get_net()
train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay)
对测试集进行分类并提交结果
net, preds = get_net(), []
train(net, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period,
lr_decay)
for X, _ in test_iter:
y_hat = net(X.to(devices[0]))
preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())
sorted_ids = list(range(1, len(test_ds) + 1))
sorted_ids.sort(key=lambda x: str(x))
df = pd.DataFrame({'id': sorted_ids, 'label': preds})
df['label'] = df['label'].apply(lambda x: train_valid_ds.classes[x])
df.to_csv('submission.csv', index=False)
实战Kaggle比赛：狗的品种识别(imageNet Dogs)
import os
import torch
import torchvision
from torch import nn
from d2l import torch as d2l
我们提供完整数据集的小规模样本
d2l.DATA_HUB['dog_tiny'] = (d2l.DATA_URL + 'kaggle_dog_tiny.zip',
'0cb91d09b814ecdc07b50f31f8dcad3e81d6a86d')
demo = True
if demo:
data_dir = d2l.download_extract('dog_tiny')
else:
data_dir = os.path.join('..', 'data', 'dog-breed-identification')
Downloading ../data/kaggle_dog_tiny.zip from http://d2l-data.s3-accelerate.amazonaws.com/kaggle_dog_tiny.zip...
整理数据
def reorg_dog_data(data_dir, valid_ratio):
labels = d2l.read_csv_labels(os.path.join(data_dir, 'labels.csv'))
d2l.reorg_train_valid(data_dir, labels, valid_ratio)
d2l.reorg_test(data_dir)
batch_size = 32 if demo else 128
valid_ratio = 0.1
reorg_dog_data(data_dir, valid_ratio)
图片增广
transform_train = torchvision.transforms.Compose([
torchvision.transforms.RandomResizedCrop(224, scale=(0.08, 1.0),
ratio=(3.0 / 4.0, 4.0 / 3.0)),
torchvision.transforms.RandomHorizontalFlip(),
torchvision.transforms.ColorJitter(brightness=0.4, contrast=0.4,
saturation=0.4), # 图像的明亮度等等
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])])
transform_test = torchvision.transforms.Compose([
torchvision.transforms.Resize(256),
torchvision.transforms.CenterCrop(224), # 从中心点copy一个224*224的图片
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])])
train_ds, train_valid_ds = [
torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_train) for folder in ['train', 'train_valid']]
valid_ds, test_ds = [
torchvision.datasets.ImageFolder(
os.path.join(data_dir, 'train_valid_test', folder),
transform=transform_test) for folder in ['valid', 'test']]
train_iter, train_valid_iter = [
torch.utils.data.DataLoader(dataset, batch_size, shuffle=True,
drop_last=True)
for dataset in (train_ds, train_valid_ds)]
valid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,
drop_last=True)
test_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,
drop_last=False)
微调预训练模型
# 这个函数的作用就是将除了最后一层以外的参数不变，拿过来
def get_net(devices):
finetune_net = nn.Sequential()
finetune_net.features = torchvision.models.resnet34(pretrained=True)
finetune_net.output_new = nn.Sequential(nn.Linear(1000, 256), nn.ReLU(),
nn.Linear(256, 120)) # 这里我们在原来的输出后面加了几层
finetune_net = finetune_net.to(devices[0])
for param in finetune_net.features.parameters():
param.requires_grad = False # 这里的意思就是将卷积层的参数固定住，不更新他了，所以设置为false
return finetune_net
计算损失
loss = nn.CrossEntropyLoss(reduction='none')
def evaluate_loss(data_iter, net, devices):
l_sum, n = 0.0, 0
for features, labels in data_iter:
features, labels = features.to(devices[0]), labels.to(devices[0])
outputs = net(features)
l = loss(outputs, labels)
l_sum += l.sum()
n += labels.numel()
return l_sum / n
训练函数
def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay):
net = nn.DataParallel(net, device_ids=devices).to(devices[0])
trainer = torch.optim.SGD(
(param for param in net.parameters() if param.requires_grad), lr=lr,
momentum=0.9, weight_decay=wd) # 这里第一个参数的肆意就是将网络中需要更新的参数给他，不需要更新的参数我们就不更新了
scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay)
num_batches, timer = len(train_iter), d2l.Timer()
legend = ['train loss']
if valid_iter is not None:
legend.append('valid loss')
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
legend=legend)
for epoch in range(num_epochs):
metric = d2l.Accumulator(2)
for i, (features, labels) in enumerate(train_iter):
timer.start()
features, labels = features.to(devices[0]), labels.to(devices[0])
trainer.zero_grad()
output = net(features)
l = loss(output, labels).sum()
l.backward()
trainer.step()
metric.add(l, labels.shape[0])
timer.stop()
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[1], None))
measures = f'train loss {metric[0] / metric[1]:.3f}'
if valid_iter is not None:
valid_loss = evaluate_loss(valid_iter, net, devices)
animator.add(epoch + 1, (None, valid_loss.detach()))
scheduler.step()
if valid_iter is not None:
measures += f', valid loss {valid_loss:.3f}'
print(measures + f'\n{metric[1] * num_epochs / timer.sum():.1f}'
f' examples/sec on {str(devices)}')
训练和验证模型
devices, num_epochs, lr, wd = d2l.try_all_gpus(), 10, 1e-4, 1e-4
lr_period, lr_decay, net = 2, 0.9, get_net(devices)
train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,
lr_decay)
对测试集分类
net = get_net(devices)
train(net, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period,
lr_decay)
preds = []
for data, label in test_iter:
output = torch.nn.functional.softmax(net(data.to(devices[0])), dim=0)
preds.extend(output.cpu().detach().numpy())
ids = sorted(
os.listdir(os.path.join(data_dir, 'train_valid_test', 'test', 'unknown')))
with open('submission.csv', 'w') as f:
f.write('id,' + ','.join(train_valid_ds.classes) + '\n')
for i, output in zip(ids, preds):
f.write(
i.split('.')[0] + ',' + ','.join([str(num)
for num in output]) + '\n')