kaggle项目实战——猫狗分类检测
主要参考：《深度学习——caffe之经典模型详解与实战》
kaggle数据集下载链接：https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data
数据集的说明：
一共由两个文件：
train.zip:训练集，图片文件命名格式：cat.X.jpg,dog.X.jpg，这个是一个二分类问题，需要根据cat/dog的类别进行0/1分类，即将cat->0,dog->1
test.zip：用来进行检测,命名格式：X.jpg，用来验证算法的识别准确率
step1:将cat_dog进行分类：
根据cat/dog的类别进行0/1分类，即将cat->0,dog->1
# -*- coding:utf-8 -*-
__author__ = 'xuy'
import commands
import os
import re
import random
my_project='/home/xuy/桌面/code/python/caffe_code/kaggle_dogcat_classification/'
train_root_dir='dataset/train'
test_root_dir='dataset/test'
train_save_name='mytrain.txt'
fw_train=open(train_save_name,'w')#打开的是文件名称
for parent,dirnames,filenames in os.walk(train_root_dir):#三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
#开始遍历train文件夹下面的所有的文件名称
for filename in filenames:
train_all_file_name = os.path.join(parent, filename) # 添加了全路径名称
if filename[:3]=='cat':
label=0
train_writeFile_content=train_all_file_name+' '+str(label)+'\n'
# fw_train.write(train_writeFile_content)
elif filename[:3]=='dog':
label=1
train_writeFile_content=train_all_file_name+' '+str(label)+'\n'
fw_train.write(train_writeFile_content)
fw_train.close()
# -*- coding: UTF-8 -*-
__author__ = 'xuy'
import commands
import os
import re
import caffe
caffe_root='/home/xuy/caffe/'
'''
我在txt文件里面已经写好了train,val的路径，train/0/***.jpg or val/***.jpg
'''
my_project_path='/home/xuy/桌面/code/python/caffe_code/kaggle_dogcat_classification/'
images_train_path=my_project_path+'dataset/train/'
# print images_train_path
txt_train_path=my_project_path+'mytrain.txt'
def create_lmdb(caffe_root, images_path, txt_save_path,lmdb_save_path):
#convert_imageset工具路径,这个工具是caffe系统自带的，
# 因此路径是caffe_root,利用命令，将txt文件中的照片文件名转化为mdb格式
convert_imageset_path = caffe_root + 'build/tools/convert_imageset'
cmd = """%s --shuffle --resize_height=32 --resize_width=32 %s %s %s"""#对于这4个%s进行说明：第一个：convert_imageset(caffe_home里面自带的工具，需要指明该工具的路径)，第二个：图片的来源，这里是至train和val文件夹下面的路径，第三个：生成的train.txt以及val.txt这两个文件，第四个：lmdb保存路径
status, output = commands.getstatusoutput(cmd % (convert_imageset_path, images_path,
txt_save_path, lmdb_save_path))
print output
if(status == 0):
print "lmbd文件生成成功"
lmdb_train_path=my_project_path+'img_train.lmdb'
create_lmdb(caffe_root,my_project_path,txt_train_path,lmdb_train_path)#生成train.lmdb,第二个参数：由于train.txt文件当中已经有了dataset/train这个路径，因此第二个参数仅仅需要加上my_project这个前缀路径即可
step3:制作val_dataset,将trainset当中的20%当作valset，也就是随机在trainset当中筛选20%的图片，复制到valset文件夹当中
# -*- coding:utf-8 -*-
__author__ = 'xuy'
import os
import shutil
import random
root_dir = '/home/xuy/桌面/code/python/caffe_code/kaggle_dogcat_classification/dataset/my_train'
output_dir = '/home/xuy/桌面/code/python/caffe_code/kaggle_dogcat_classification/dataset/val'
ref = 1
for root, dirs, files in os.walk(root_dir):
number_of_files = len(os.listdir(root))
if number_of_files > ref:
ref_copy = int(round(0.2 * number_of_files))#随机筛选20%的图片到新建的文件夹当中
for i in xrange(ref_copy):
chosen_one = random.choice(os.listdir(root))
file_in_track = root
file_to_copy = file_in_track + '/' + chosen_one
if os.path.isfile(file_to_copy) == True:
shutil.copy(file_to_copy,output_dir)
print file_to_copy
else:
for i in xrange(len(files)):
track_list = root
file_in_track = files[i]
file_to_copy = track_list + '/' + file_in_track
if os.path.isfile(file_to_copy) == True:
shutil.copy(file_to_copy,output_dir)
print file_to_copy
print 'Finished !'step4:构造神经网络：train_val.prototxt,套路和上一篇博客一样，基于pycaffe搭建神经网络
# -*- coding:utf-8 -*-
__author__ = 'xuy'
import caffe
frozen_weight_param = dict(lr_mult=1)#权重
frozen_bias_param = dict(lr_mult=2)#偏执值
froozen_param = [frozen_weight_param, frozen_bias_param]
def write_layer():
net = caffe.NetSpec()
net.data,net.label=caffe.layers.Data(
name='catdog',
include=dict(phase=caffe.TRAIN),
ntop=2,
source='/home/xuy/桌面/code/python/caffe_code/kaggle_dogcat_classification/img_train.lmdb',
backend = caffe.params.Data.LMDB,
batch_size=16,
transform_param = dict(
scale=1 / 255.
)
)
net.conv1 = caffe.layers.Convolution(
net.data,
param=froozen_param, # 这里通过定义一个list，来整合到param的字典,也就是：param=[]
num_output=32,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu1=caffe.layers.ReLU(
net.conv1,
in_place=True
)
net.conv2=caffe.layers.Convolution(
net.conv1,
param=froozen_param,
num_output=32,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu2=caffe.layers.ReLU(
net.conv2,
in_place=True
)
net.pool1=caffe.layers.Pooling(
net.conv2,
pool=caffe.params.Pooling.MAX,
kernel_size=2,
stride=2
)
net.conv3=caffe.layers.Convolution(
net.pool1,
param=froozen_param,
num_output=64,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu3 = caffe.layers.ReLU(
net.conv3,
in_place=True
)
net.conv4=caffe.layers.Convolution(
net.conv3,
param=froozen_param,
num_output=64,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu4 = caffe.layers.ReLU(
net.conv4,
in_place=True
)
net.pool2 = caffe.layers.Pooling(
net.conv4,
pool=caffe.params.Pooling.MAX,
kernel_size=2,
stride=2
)
net.conv5 = caffe.layers.Convolution(
net.pool2,
param=froozen_param,
num_output=128,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu5 = caffe.layers.ReLU(
net.conv5,
in_place=True
)
net.conv6 = caffe.layers.Convolution(
net.conv5,
param=froozen_param,
num_output=128,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu6 = caffe.layers.ReLU(
net.conv6,
in_place=True
)
net.pool3 = caffe.layers.Pooling(
net.conv6,
pool=caffe.params.Pooling.MAX,
kernel_size=2,
stride=2
)
net.conv7 = caffe.layers.Convolution(
net.pool3,
param=froozen_param,
num_output=256,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu7 = caffe.layers.ReLU(
net.conv7,
in_place=True
)
net.conv8 = caffe.layers.Convolution(
net.conv7,
param=froozen_param,
num_output=256,
pad=1,
kernel_size=3,
stride=1,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu8 = caffe.layers.ReLU(
net.conv8,
in_place=True
)
net.pool4 = caffe.layers.Pooling(
net.conv8,
pool=caffe.params.Pooling.MAX,
kernel_size=2,
stride=2
)
net.ip1=caffe.layers.InnerProduct(
net.pool4,
param=froozen_param,
num_output=256,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.drop1=caffe.layers.Dropout(
net.ip1,
in_place=True,
dropout_param=dict(dropout_ratio=0.5)
)
net.ip2=caffe.layers.InnerProduct(
net.ip1,
param=froozen_param,
num_output=256,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.relu9 = caffe.layers.ReLU(
net.ip2,
in_place=True
)
net.drop2 = caffe.layers.Dropout(
net.ip2,
in_place=True,
dropout_param=dict(dropout_ratio=0.5)
)
net.ip3=caffe.layers.InnerProduct(
net.ip2,
param=froozen_param,
num_output=2,
weight_filler=dict(type='xavier'), bias_filler=dict(type='constant')
)
net.loss=caffe.layers.SoftmaxWithLoss(net.ip3,net.label)
net.accuracy=caffe.layers.Accuracy(
net.ip3,
net.label,
include=dict(phase=caffe.TEST)
)
return net.to_proto()
with open('train.prototxt', 'w') as f:
f.write(str(write_layer()))生成的train.prototxt的内容：
layer {
name: "catdog"
type: "Data"
top: "data"
top: "label"
include {
phase: TRAIN
}
transform_param {
scale: 0.00392156885937
}
data_param {
source: "/home/xuy/\346\241\214\351\235\242/code/python/caffe_code/kaggle_dogcat_classification/img_train.lmdb"
batch_size: 16
backend: LMDB
}
}
layer {
name: "conv1"
type: "Convolution"
bottom: "data"
top: "conv1"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 32
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu1"
type: "ReLU"
bottom: "conv1"
top: "conv1"
}
layer {
name: "conv2"
type: "Convolution"
bottom: "conv1"
top: "conv2"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 32
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu2"
type: "ReLU"
bottom: "conv2"
top: "conv2"
}
layer {
name: "pool1"
type: "Pooling"
bottom: "conv2"
top: "pool1"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "conv3"
type: "Convolution"
bottom: "pool1"
top: "conv3"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 64
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu3"
type: "ReLU"
bottom: "conv3"
top: "conv3"
}
layer {
name: "conv4"
type: "Convolution"
bottom: "conv3"
top: "conv4"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 64
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu4"
type: "ReLU"
bottom: "conv4"
top: "conv4"
}
layer {
name: "pool2"
type: "Pooling"
bottom: "conv4"
top: "pool2"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "conv5"
type: "Convolution"
bottom: "pool2"
top: "conv5"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 128
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu5"
type: "ReLU"
bottom: "conv5"
top: "conv5"
}
layer {
name: "conv6"
type: "Convolution"
bottom: "conv5"
top: "conv6"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 128
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu6"
type: "ReLU"
bottom: "conv6"
top: "conv6"
}
layer {
name: "pool3"
type: "Pooling"
bottom: "conv6"
top: "pool3"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "conv7"
type: "Convolution"
bottom: "pool3"
top: "conv7"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 256
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu7"
type: "ReLU"
bottom: "conv7"
top: "conv7"
}
layer {
name: "conv8"
type: "Convolution"
bottom: "conv7"
top: "conv8"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
convolution_param {
num_output: 256
pad: 1
kernel_size: 3
stride: 1
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu8"
type: "ReLU"
bottom: "conv8"
top: "conv8"
}
layer {
name: "pool4"
type: "Pooling"
bottom: "conv8"
top: "pool4"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "ip1"
type: "InnerProduct"
bottom: "pool4"
top: "ip1"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
inner_product_param {
num_output: 256
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "drop1"
type: "Dropout"
bottom: "ip1"
top: "ip1"
dropout_param {
dropout_ratio: 0.5
}
}
layer {
name: "ip2"
type: "InnerProduct"
bottom: "ip1"
top: "ip2"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
inner_product_param {
num_output: 256
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "relu9"
type: "ReLU"
bottom: "ip2"
top: "ip2"
}
layer {
name: "drop2"
type: "Dropout"
bottom: "ip2"
top: "ip2"
dropout_param {
dropout_ratio: 0.5
}
}
layer {
name: "ip3"
type: "InnerProduct"
bottom: "ip2"
top: "ip3"
param {
lr_mult: 1.0
}
param {
lr_mult: 2.0
}
inner_product_param {
num_output: 2
weight_filler {
type: "xavier"
}
bias_filler {
type: "constant"
}
}
}
layer {
name: "loss"
type: "SoftmaxWithLoss"
bottom: "ip3"
bottom: "label"
top: "loss"
}
layer {
name: "accuracy"
type: "Accuracy"
bottom: "ip3"
bottom: "label"
top: "accuracy"
include {
phase: TEST
}
}
最后附上solver.prototxt的定义：
net:"你的路径"
test_iter:50#test.prototxt的batch_size是16，因此每进行一次epoth，读取50*16=800张图片
test_interval:500#每进行500次训练进行一次测试
base_lr:0.0001#推荐看2018年fast ai当中的视频，find_lr()函数可以找出最优的lr
momentum:0.9
weight_decay:0.0005
lr_policy:"fixed"
gamma:0.0001
power:0.75
display:100#每100次迭代显示一次
max_iter:100000
snapshot:5000
snapshot_prefix:example/catdog/catdog
solver_mode:GPU