Kaggle : Using a Convolutional Neural Network for classifying Cats vs Dogs
数据下载
https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data
Part 1 - Preprocessing
#Package Requirements
#!/usr/bin/python2
# -*- coding: UTF-8 -*-
import cv2 # working with, mainly resizing, images
import numpy as np # dealing with arrays
import os # dealing with directories
from random import shuffle # mixing up or currently ordered data that might lead our network astray in training.
from tqdm import tqdm # a nice pretty percentage bar for tasks.
TRAIN_DIR = '/home/bids/miscellaneous/catvsdog/train'
TEST_DIR = '/home/bids/miscellaneous/catvsdog/test'
IMG_SIZE = 50
LR = 1e-3
MODEL_NAME = 'dogsvscats-{}-{}.model'.format(LR, '2conv-basic') # just so we remember which saved model is which, sizes must match
#Convert the dog/cat to an array with one-hot，标签类别化
def label_img(img):
word_label = img.split('.')[-3]
# conversion to one-hot array [cat,dog]
# [much cat, no dog]
if word_label == 'cat': return [1,0]
# [no cat, very doggo]
elif word_label == 'dog': return [0,1]
#fully process the training images and their labels into arrays训练集预处理
def create_train_data():
training_data = []
for img in tqdm(os.listdir(TRAIN_DIR)):
label = label_img(img)
path = os.path.join(TRAIN_DIR,img)
img = cv2.imread(path,cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))
training_data.append([np.array(img),np.array(label)])
shuffle(training_data) #将序列的所有元素随机排序
np.save('train_data.npy', training_data)
return training_data
#无标签数据预处理
def process_test_data():
testing_data = []
for img in tqdm(os.listdir(TEST_DIR)):
path = os.path.join(TEST_DIR,img)
img_num = img.split('.')[0]
img = cv2.imread(path,cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))
testing_data.append([np.array(img), img_num])
shuffle(testing_data)
np.save('test_data.npy', testing_data)
return testing_data
#run the training生成训练集合
train_data = create_train_data()
# If you have already created the dataset:
#train_data = np.load('train_data.npy')Part 2 - Convolutional Neural Network
#define our neural network
import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
convnet = input_data(shape=[None, IMG_SIZE, IMG_SIZE, 1], name='input')
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 2, activation='softmax')
convnet = regression(convnet, optimizer='adam', learning_rate=LR, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet, tensorboard_dir='log')
#saving our model after every session, and reloading it
if os.path.exists('{}.meta'.format(MODEL_NAME)):
model.load(MODEL_NAME)
print('model loaded!')
#split out training and testing data，有标签数据分为训练集和测试集
train = train_data[:-500]
test = train_data[-500:]
#separate my features and labels 特征，类别分离
X = np.array([i[0] for i in train]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
Y = [i[1] for i in train]
test_x = np.array([i[0] for i in test]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
test_y = [i[1] for i in test]
#fit for 3 epochs
model.fit({'input': X}, {'targets': Y}, n_epoch=3, validation_set=({'input': test_x}, {'targets': test_y}),
snapshot_step=500, show_metric=True, run_id=MODEL_NAME)
执行结果
Training Step: 1148 | total loss: 11.71334 | time: 4.061s
| Adam | epoch: 003 | loss: 11.71334 - acc: 0.4913 -- iter: 24448/24500
Training Step: 1149 | total loss: 11.72928 | time: 5.074s
| Adam | epoch: 003 | loss: 11.72928 - acc: 0.4906 | val_loss: 11.88134 - val_acc: 0.4840 -- iter: 24500/24500
--#Part 3 - Size Matters
＃探讨网络深度的影响
import tensorflow as tf
tf.reset_default_graph()
convnet = input_data(shape=[None, IMG_SIZE, IMG_SIZE, 1], name='input')
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 128, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 2, activation='softmax')
convnet = regression(convnet, optimizer='adam', learning_rate=LR, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet, tensorboard_dir='log')
if os.path.exists('{}.meta'.format(MODEL_NAME)):
model.load(MODEL_NAME)
print('model loaded!')
train = train_data[:-500]
test = train_data[-500:]
X = np.array([i[0] for i in train]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
Y = [i[1] for i in train]
test_x = np.array([i[0] for i in test]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
test_y = [i[1] for i in test]
model.fit({'input': X}, {'targets': Y}, n_epoch=20, validation_set=({'input': test_x}, {'targets': test_y}),
snapshot_step=500, show_metric=True, run_id=MODEL_NAME)
执行结果
Training Step: 7659 | total loss: 0.30021 | time: 7.384s
| Adam | epoch: 020 | loss: 0.30021 - acc: 0.8845 -- iter: 24448/24500
Training Step: 7660 | total loss: 0.29932 | time: 8.405s
| Adam | epoch: 020 | loss: 0.29932 - acc: 0.8820 | val_loss: 0.53824 - val_acc: 0.7900 -- iter: 24500/24500
--# save the model保存模型
model.save(MODEL_NAME)#reload the model and continue training载入模型继续训练
import tensorflow as tf
tf.reset_default_graph()
convnet = input_data(shape=[None, IMG_SIZE, IMG_SIZE, 1], name='input')
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 128, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 2, activation='softmax')
convnet = regression(convnet, optimizer='adam', learning_rate=LR, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet, tensorboard_dir='log')
if os.path.exists('/home/bids/miscellaneous/catvsdog/{}.meta'.format(MODEL_NAME)):
model.load(MODEL_NAME)
print('model loaded!')
train = train_data[:-500]
test = train_data[-500:]
X = np.array([i[0] for i in train]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
Y = [i[1] for i in train]
test_x = np.array([i[0] for i in test]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
test_y = [i[1] for i in test]
model.fit({'input': X}, {'targets': Y}, n_epoch=10, validation_set=({'input': test_x}, {'targets': test_y}),
snapshot_step=500, show_metric=True, run_id=MODEL_NAME)
model.save(MODEL_NAME)执行结果
Training Step: 11489 | total loss: 0.15848 | time: 6.454s
| Adam | epoch: 010 | loss: 0.15848 - acc: 0.9387 -- iter: 24448/24500
Training Step: 11490 | total loss: 0.15471 | time: 7.476s
| Adam | epoch: 010 | loss: 0.15471 - acc: 0.9385 | val_loss: 1.08745 - val_acc: 0.7840 -- iter: 24500/24500
--
INFO:tensorflow:/home/bids/miscellaneous/catvsdog/dogsvscats-0.001-2conv-basic.model is not in all_model_checkpoint_paths. Manually adding it.Part 4 - Visually inspecting our network against unlabeled data
import matplotlib.pyplot as plt
# if you need to create the data:
#test_data = process_test_data()
# if you already have some saved:
test_data = np.load('test_data.npy')
fig=plt.figure()
for num,data in enumerate(test_data[:12]):
# cat: [1,0]
# dog: [0,1]
img_num = data[1]
img_data = data[0]
y = fig.add_subplot(3,4,num+1)
orig = img_data
data = img_data.reshape(IMG_SIZE,IMG_SIZE,1)
#model_out = model.predict([data])[0]
model_out = model.predict([data])[0]
if np.argmax(model_out) == 1: str_label='Dog'
else: str_label='Cat'
y.imshow(orig,cmap='gray')
plt.title(str_label)
y.axes.get_xaxis().set_visible(False)
y.axes.get_yaxis().set_visible(False)
plt.show()
Part 5 - Submit
import time
start = time.clock()
with open('submission_file.csv','w') as f:
f.write('id,prolabel0,prolabel1,labelname\n')
with open('submission_file.csv','a') as f:
for data in tqdm(test_data):
img_num = data[1]
img_data = data[0]
orig = img_data
data = img_data.reshape(IMG_SIZE,IMG_SIZE,1)
model_out = model.predict([data])[0]
#对概率最大的值赋予标签名
if np.argmax(model_out) == 1:
str_label='Dog'
else:
str_label='Cat'
f.write('{},{},{},{}\n'.format(img_num,model_out[0],model_out[1],str_label))
end = time.clock()
print("The running time is %g s" %(end-start))执行结果及显示
The running time is 50.5271 ssubmission_file.csv
id,prolabel0,prolabel1,labelname
1212,0.921218276024,0.0787817165256,Cat
9302,1.18830442375e-07,0.999999880791,Dog
12411,1.42178024021e-07,0.999999880791,Dog
6441,0.083547629416,0.916452348232,Dog
7080,0.427787721157,0.572212278843,Dog
5382,0.00636592926458,0.993634104729,Dog
353,0.782497525215,0.217502549291,Cat
5446,6.87951469445e-05,0.99993121624,Dog
4448,0.653192222118,0.346807777882,Cat
338,0.999975681305,2.43726990448e-05,Cat
9638,0.989276170731,0.010723865591,Cat
3980,0.998381376266,0.00161859672517,Cat
7844,0.608684241772,0.391315758228,Cat
9771,4.93591596751e-06,0.999995112419,Dog
10791,0.669038891792,0.330961108208,Cat
3925,0.0286846794188,0.971315264702,Dog
7400,0.246031478047,0.753968536854,Dog
5971,0.0695387497544,0.930461287498,Dog
3286,0.999945521355,5.44977847312e-05,Cat
6044,0.971816003323,0.0281839352101,Cat
9298,0.0108342785388,0.989165723324,Dog
729,0.999630212784,0.000369725632481,Cat
2853,1.0,1.77208714369e-09,Cat
516,0.0357219539583,0.964278101921,Dog
5129,0.997949063778,0.00205094460398,Cat
10121,0.999883055687,0.000116942981549,Cat
3133,0.00119872530922,0.998801231384,Dog
4034,0.999985337257,1.46087259054e-05,Cat
3158,0.999208390713,0.000791579775978,Cat
4069,0.307045787573,0.692954182625,Dog
1478,1.0,2.06947931014e-11,Cat
7171,0.999999880791,9.28167693814e-08,Cat
10217,0.394322454929,0.605677485466,Dog
7977,0.178869321942,0.821130692959,Dog
9163,0.999999523163,4.64977688353e-07,Cat
11553,0.866536676884,0.133463352919,Cat
6130,0.997319757938,0.00268025626428,Cat
3611,1.0,1.71961218598e-08,Cat
1042,0.474040418863,0.525959551334,Dog
8793,0.352943599224,0.647056400776,Dog
9708,3.69108086318e-08,1.0,Dog
3499,1.0,4.09031670756e-08,Cat
2291,0.148211851716,0.85178810358,Dog
8138,0.391484260559,0.608515799046,Dog
9219,0.998136520386,0.00186341057997,Cat
11983,0.885705888271,0.114294171333,Cat
3192,0.18194848299,0.81805151701,Dog
3734,7.41316398489e-07,0.999999284744,Dog
4732,0.00249696546234,0.997503101826,Dog
10883,4.8458132369e-06,0.999995112419,Dog
11081,1.06594075078e-06,0.999998927116,Dog
11917,0.999524831772,0.000475237378851,Cat
2637,0.944479882717,0.0555201396346,Cat
7868,0.872499227524,0.127500742674,Cat
11800,0.999999642372,3.36826360581e-07,Cat
3444,0.990353822708,0.00964620243758,Cat
5415,0.990461647511,0.00953831989318,Cat
11853,0.958735942841,0.0412640571594,Cat
880,0.99071675539,0.0092832185328,Cat
8928,0.994742512703,0.00525750452653,Cat
12410,0.653257966042,0.346742123365,Cat
2755,0.443003803492,0.556996226311,Dog
9677,2.42542625983e-07,0.999999761581,Dog
9417,2.07065786739e-09,1.0,Dog
8962,0.969185173512,0.0308148153126,Cat
4797,1.27675070871e-06,0.999998688698,Dog
2027,0.341440081596,0.658559858799,Dog
11619,0.979589402676,0.0204106215388,Cat
8100,0.999998807907,1.18297475638e-06,Cat
10599,0.0924441888928,0.907555758953,Dog
10758,0.0891051217914,0.910894930363,Dog
1497,0.000189088241314,0.999810993671,Dog
11472,0.00835430901498,0.991645753384,Dog
6861,0.931414544582,0.0685854256153,Cat
12459,0.000314021250233,0.999686002731,Dog
4517,0.999940514565,5.95222300035e-05,Cat
4550,0.928105473518,0.0718945041299,Cat
12385,0.00242776027881,0.997572243214,Dog
4907,0.987588703632,0.0124113131315,Cat
10130,0.881314516068,0.118685476482,Cat
3146,0.999999523163,4.93570894378e-07,Cat
3786,0.997477352619,0.00252265273593,Cat
1203,0.0800434201956,0.919956505299,Dog
2496,0.415617018938,0.58438295126,Dog
7563,0.95674598217,0.043254006654,Cat
2239,0.970162272453,0.0298377349973,Cat
4682,0.999291062355,0.000708963314537,Cat
4378,0.0031416725833,0.996858358383,Dog
7665,0.909289181232,0.0907108262181,Cat
2823,0.99922311306,0.000776931643486,Cat
2185,0.000722112716176,0.999277889729,Dog
5316,0.583464622498,0.416535377502,Cat
9270,0.468584805727,0.531415224075,Dog
11067,0.953469336033,0.0465307161212,Cat
5680,0.125824138522,0.874175846577,Dog
9933,0.609232544899,0.390767514706,Cat
4388,0.00831792596728,0.991682112217,Dog
6527,0.316535592079,0.683464348316,Dog
8562,0.0318263992667,0.96817356348,Dog
3506,0.718664944172,0.281334996223,Cat
1401,0.00189117959235,0.998108863831,Dog
100,0.99985408783,0.000145845173392,Cat
3225,0.308728516102,0.691271424294,Dog
9353,0.941712677479,0.0582873113453,Cat
7416,0.25361225009,0.746387779713,Dog
841,0.836331427097,0.16366866231,Cat
3107,0.0021977883298,0.997802197933,Dog
10709,1.39225221574e-05,0.999986052513,Dog
6230,0.00928914453834,0.990710794926,Dog
10398,0.999177634716,0.000822360860184,Cat
2232,0.592651724815,0.407348304987,Cat
8878,0.0124056786299,0.987594246864,Dog
408,0.203185468912,0.796814501286,Dog
3679,0.952772557735,0.0472274385393,Cat
3445,8.16722513264e-06,0.999991774559,Dog
7682,5.48541611352e-05,0.999945163727,Dog
11807,0.982504308224,0.0174957159907,Cat
11197,0.97709685564,0.0229031760246,Cat
7556,0.113571561873,0.886428415775,Dog
5190,0.999565899372,0.000434125337051,Cat
10039,0.163930222392,0.836069762707,Dog
9409,3.1199199384e-06,0.999996900558,Dog
3684,0.0231008175761,0.976899206638,Dog
3436,0.0557881891727,0.944211781025,Dog
6475,0.000631944218185,0.999368131161,Dog
945,0.0242063160986,0.975793719292,Dog
12042,0.834796190262,0.165203869343,Cat
……
……
……
参考文献
Classifying Cats vs Dogs with a Convolutional Neural Network on Kaggle