Caffe学习笔记（一）: 训练和测试自己的数据集
1 数据准备
首先在caffe根目录下建立一个文件夹myfile，用于存放数据文件和后面的caffe模型相关文件。
然后在myfile文件夹下建立build_lmdb和datatest两个文件夹，其中build_lmdb文件夹用于存放生成的lmdb文件，datatest文件夹存放图片数据。
在datatest下主要有2个文件夹和2个.sh文件和2个.txt文件，其中train文件夹中存放待训练的图片，val文件夹中存放待测试的图片，creatlabel.py是分别用来生成train和val的.py文件，train.txt和val.txt是生成的标签文件。
建立好文件夹后，我们把分割好的图片放到train和val文件夹里面
首先生成label
# -*- coding: utf-8 -*-
import csv
import os
import numpy as np
def IsSubString(SubStrList,Str):
'''''
#判断字符串Str是否包含序列SubStrList中的每一个子字符串
#>>>SubStrList=['F','EMS','txt']
#>>>Str='F06925EMS91.txt'
#>>>IsSubString(SubStrList,Str)#return True (or False)
'''
flag=True
for substr in SubStrList:
if not(substr in Str):
flag=False
return flag
#~ #----------------------------------------------------------------------
def GetFileList(FindPath,FlagStr=[]):
FileList=[]
FileNames=os.listdir(FindPath)
if (len(FileNames)>0):
for fn in FileNames:
if (len(FlagStr)>0):
#返回指定类型的文件名
if (IsSubString(FlagStr,fn)):
fullfilename=os.path.join(FindPath,fn)
FileList.append(fullfilename)
else:
#默认直接返回所有文件名
fullfilename=os.path.join(FindPath,fn)
FileList.append(fullfilename)
#对文件名排序
if (len(FileList)>0):
FileList.sort()
return FileList
if __name__=="__main__":
FileNameList=GetFileList("/home/user/swscode/datatest/train/",[])
print FileNameList
# fileObject = open('val.txt', 'w')
data = np.array([FileNameList, ])
labellist = []
for ip in FileNameList:
print ip
# fileObject.write(ip)
# fileObject.write('\n')
iq=ip.split('/')[-1].split("_")[0]
iq = int(iq)
iq = iq-2
iq=str(iq)
labellist.append(iq)
with open('train.csv', 'w') as f:
print "test"
writer = csv.writer(f, delimiter='\t')
writer.writerows(zip(FileNameList,labellist))
# fileObject.close()2 生成train.txt val.txt
数据按8:2比例分配到train和val文件夹
遇到的问题：输出比例错误
解决方案：用字典计数。每一类label按比例分离，而不是整体按比例分离。
遇到的问题：两列label之间多了tab导致找不到文件
解决方案：:%s/\t/ /g 全局替换
# -*- coding: utf-8 -*-
import shutil
import csv
s =0
diriq = {}
# with open('text.csv', 'r') as f:
Filenamelist =[]
labellist=[]
import csv
with open('text.csv') as csvfile:
readCSV = csv.reader(csvfile, delimiter=',')
for row in readCSV:
print row
print row[0].split()
Filenamelist.append(row[0].split()[0])
labellist.append(row[0].split()[1])
# print row[1]
# a1 = [row for row in DictReader(f)]
# print "--->", type(a1)
# print len(a1)
# a2 = [row["2"] for row in DictReader(f)]
for a in labellist :
if a in diriq:
diriq[a]+=1
else:
diriq[a]=1
count ={}
for file, idname in zip(Filenamelist,labellist):
if count.has_key(idname):
if count[idname] < diriq[idname]*0.8:
print file
count[idname]+=1
shutil.copy(file,'/home/user/swscode/datatest/train/')
else:
shutil.copy(file,'/home/user/swscode/datatest/val/')
else:
count[idname]=1
shutil.copy(file,'/home/user/swscode/datatest/train/')3 生成lmdb数据
将图片文件转换成caffe框架中能直接使用的db文件。
遇到的问题：数据路径TRAIN_DATA_ROOT写的绝对路径，导致路径重叠，以至找不到文件
解决方案：路径改为当前路径TRAIN_DATA_ROOT="/"
# Create the imagenet lmdb inputs
# N.B. set the path to the imagenet train + val data dirs
TOOLS=/home/user/swfcode/caffe/build/tools/
EXAMPLE=/home/user/swscode/build_lmdb/
DATA=/home/user/swscode/datatest/
TRAIN_DATA_ROOT="/"
VAL_DATA_ROOT="/"
# Set RESIZE=true to resize the images to 256x256. Leave as false if images have
# already been resized using another tool.
RESIZE=true
if $RESIZE; then
RESIZE_HEIGHT=256
RESIZE_WIDTH=256
else
RESIZE_HEIGHT=0
RESIZE_WIDTH=0
fi
# if [ ! -d "$TRAIN_DATA_ROOT" ]; then
# echo "Error: TRAIN_DATA_ROOT is not a path to a directory: $TRAIN_DATA_ROOT"
# echo "Set the TRAIN_DATA_ROOT variable in create_imagenet.sh to the path" \
# "where the ImageNet training data is stored."
# exit 1
# fi
# if [ ! -d "$VAL_DATA_ROOT" ]; then
# echo "Error: VAL_DATA_ROOT is not a path to a directory: $VAL_DATA_ROOT"
# echo "Set the VAL_DATA_ROOT variable in create_imagenet.sh to the path" \
# "where the ImageNet validation data is stored."
# exit 1
# fi
echo "Creating train lmdb..."
# --backend="leveldb"\
GLOG_logtostderr=1 $TOOLS/convert_imageset \
--resize_height=$RESIZE_HEIGHT \
--resize_width=$RESIZE_WIDTH \
--shuffle \
$TRAIN_DATA_ROOT \
./train.txt \
$EXAMPLE/"patch"${1}"_train_lmdb"
echo "Creating val lmdb..."
# --backend="leveldb"\
GLOG_logtostderr=1 $TOOLS/convert_imageset \
--resize_height=$RESIZE_HEIGHT \
--resize_width=$RESIZE_WIDTH \
--shuffle \
$VAL_DATA_ROOT \
./val.txt \
$EXAMPLE/"patch"${1}"_val_lmdb"
echo "Done."4 计算均值并保存
myfile中新建文件create_meanfile.sh。图片减去均值后，再进行训练和测试，会提高速度和精度。
使用./build/tools/compute_image_mean工具根据/examples/cifar10/cifar10_train_lmdb(刚刚生成的)文件计算出这些训练集图片数据的均值，并将其保存到/examples/cifar10/mean.binaryproto文件中。
遇到的问题：数据路径重叠，以至找不到文件
解决方案：DATA="./ ”
#!/bin/bash
EXAMPLE=/home/user/swscode/build_lmdb/
DATA="./"
TOOLS=/home/user/swfcode/caffe/build/tools
$TOOLS/compute_image_mean $EXAMPLE/train_lmdb \
imagenet_mean.binaryproto
echo "Done." 5 创建模型并编写配置文件
在myfile4中创建myfile4_train_test.prototxt文件
创建myfile4_solver.prototxt文件
根据examples/cifar10/cifar10_quick_solver.prototxt这个配置文件建立模型。名为cifar10_quick_solver的CNN模型由卷基层(convolution)、池化层(pooling)、非线性ReLU层(rectified linear unit (ReLU) nonlinearities)和在顶端的局部对比归一化线性分类器组成(local contrast normalization with a linear classifier on top of it all)。
name: "ResNet-50"
layer {
name: "data"
type: "Data"
top: "data"
top: "label"
include {
phase: TRAIN
}
transform_param {
mirror: true
crop_size: 224
mean_file: "/home/user/swscode/imagenet_mean.binaryproto"
}
data_param {
source: "/home/user/swscode/build_lmdb/train_lmdb"
batch_size: 32
backend: LMDB
}
}
layer {
name: "data"
type: "Data"
top: "data"
top: "label"
include {
phase: TEST
}
transform_param {
mirror: false
crop_size: 224
mean_file: "/home/user/swscode/imagenet_mean.binaryproto"
}
data_param {
source: "/home/user/swscode/build_lmdb/val_lmdb"
batch_size: 10
backend: LMDB
}
}
layer {
bottom: "data"
top: "conv1"
name: "conv1"
type: "Convolution"
convolution_param {
num_output: 64
kernel_size: 7
pad: 3
stride: 2
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "conv1"
top: "conv1"
name: "bn_conv1"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "conv1"
top: "conv1"
name: "scale_conv1"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "conv1"
top: "conv1"
name: "conv1_relu"
type: "ReLU"
}
layer {
bottom: "conv1"
top: "pool1"
name: "pool1"
type: "Pooling"
pooling_param {
kernel_size: 3
stride: 2
pool: MAX
}
}
layer {
bottom: "pool1"
top: "res2a_branch1"
name: "res2a_branch1"
type: "Convolution"
convolution_param {
num_output: 256
kernel_size: 1
pad: 0
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res2a_branch1"
top: "res2a_branch1"
name: "bn2a_branch1"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res2a_branch1"
top: "res2a_branch1"
name: "scale2a_branch1"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "pool1"
top: "res2a_branch2a"
name: "res2a_branch2a"
type: "Convolution"
convolution_param {
num_output: 64
kernel_size: 1
pad: 0
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res2a_branch2a"
top: "res2a_branch2a"
name: "bn2a_branch2a"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res2a_branch2a"
top: "res2a_branch2a"
name: "scale2a_branch2a"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res2a_branch2a"
top: "res2a_branch2a"
name: "res2a_branch2a_relu"
type: "ReLU"
}
layer {
bottom: "res2a_branch2a"
top: "res2a_branch2b"
name: "res2a_branch2b"
type: "Convolution"
convolution_param {
num_output: 64
kernel_size: 3
pad: 1
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res2a_branch2b"
top: "res2a_branch2b"
name: "bn2a_branch2b"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res2a_branch2b"
top: "res2a_branch2b"
name: "scale2a_branch2b"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res2a_branch2b"
top: "res2a_branch2b"
name: "res2a_branch2b_relu"
type: "ReLU"
}
layer {
bottom: "res2a_branch2b"
top: "res2a_branch2c"
name: "res2a_branch2c"
type: "Convolution"
convolution_param {
num_output: 256
kernel_size: 1
pad: 0
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res2a_branch2c"
top: "res2a_branch2c"
name: "bn2a_branch2c"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res2a_branch2c"
top: "res2a_branch2c"
name: "scale2a_branch2c"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res2a_branch1"
bottom: "res2a_branch2c"
top: "res2a"
name: "res2a"
type: "Eltwise"
}
layer {
bottom: "res2a"
top: "res2a"
name: "res2a_relu"
type: "ReLU"
}
...
layer {
bottom: "res5c_branch2a"
top: "res5c_branch2a"
name: "bn5c_branch2a"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res5c_branch2a"
top: "res5c_branch2a"
name: "scale5c_branch2a"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res5c_branch2a"
top: "res5c_branch2a"
name: "res5c_branch2a_relu"
type: "ReLU"
}
layer {
bottom: "res5c_branch2a"
top: "res5c_branch2b"
name: "res5c_branch2b"
type: "Convolution"
convolution_param {
num_output: 512
kernel_size: 3
pad: 1
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res5c_branch2b"
top: "res5c_branch2b"
name: "bn5c_branch2b"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res5c_branch2b"
top: "res5c_branch2b"
name: "scale5c_branch2b"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res5c_branch2b"
top: "res5c_branch2b"
name: "res5c_branch2b_relu"
type: "ReLU"
}
layer {
bottom: "res5c_branch2b"
top: "res5c_branch2c"
name: "res5c_branch2c"
type: "Convolution"
convolution_param {
num_output: 2048
kernel_size: 1
pad: 0
stride: 1
bias_term: false
weight_filler {
type: "msra"
}
}
}
layer {
bottom: "res5c_branch2c"
top: "res5c_branch2c"
name: "bn5c_branch2c"
type: "BatchNorm"
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
param {
lr_mult: 0
decay_mult: 0
}
}
layer {
bottom: "res5c_branch2c"
top: "res5c_branch2c"
name: "scale5c_branch2c"
type: "Scale"
scale_param {
bias_term: true
}
}
layer {
bottom: "res5b"
bottom: "res5c_branch2c"
top: "res5c"
name: "res5c"
type: "Eltwise"
}
layer {
bottom: "res5c"
top: "res5c"
name: "res5c_relu"
type: "ReLU"
}
layer {
bottom: "res5c"
top: "pool5"
name: "pool5"
type: "Pooling"
pooling_param {
kernel_size: 7
stride: 1
pool: AVE
}
}
layer {
bottom: "pool5"
top: "fc1500"
name: "fc1500"
type: "InnerProduct"
inner_product_param {
num_output: 1500
weight_filler {
type: "msra"
}
bias_filler {
type: "constant"
value: 0
}
}
}
layer {
bottom: "fc1500"
bottom: "label"
top: "prob"
name: "prob"
type: "SoftmaxWithLoss"
include {
phase: TRAIN
}
}
layer {
bottom: "fc1500"
bottom: "label"
top: "accuracy@1"
name: "accuracy/top1"
type: "Accuracy"
accuracy_param {
top_k: 1
}
}
layer {
bottom: "fc1500"
bottom: "label"
top: "accuracy@5"
name: "accuracy/top5"
type: "Accuracy"
accuracy_param {
top_k: 5
}
}6 训练和测试
# ./build/tools/caffe train -solver examples/myfile/myfile_solver.prototxt -weights models/bvlc_reference_caffenet/ResNet-50-model.caffemodel -gpu 1
7 用训练好的模型进行分类
遇到的问题：训练结果都是0.01
解决方案：改deploy里的全连接层类别数以及num_ output 为1500
遇到的问题：找不到caffe里的库函数
解决的方案：sys.path.insert(0,caffe_root+'python')
小结：网络训练模型train_test.prototxt 测试部署模型deploy.prototxt 求解参数solver.prototxt
#coding=utf-8
import sys
sys.path.insert(0,'/home/user/swfcode/caffe/python')
import os
import caffe
import numpy as np
root='/home/user/swscode/datatest/'
deploy=root +'ResNet-50-deploy.prototxt'
caffe_model='/data1/swfdata/swfmodels/residualnet_imagenet/sws_1211resnet_50_iter_150000.caffemodel'
import os
import numpy as np
mylist=[]
dir = root+'val/'
filelist=[]
filenames = os.listdir(dir)
for fn in filenames:
fullfilename = os.path.join(dir,fn)
filelist.append(fullfilename)
net = caffe.Net(deploy,caffe_model,caffe.TEST)
# img=root+‘data/DRIVE/test/60337.jpg‘
def Test(img, net):
# transformer = caffe.io.Transformer({'data':net.blobs['data'].data.shape})
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2,0,1))
#transformer.set_mean(‘data‘, np.load(mean_file).mean(1).mean(1))
transformer.set_raw_scale('data', 255)
transformer.set_channel_swap('data', (2,1,0))
im=caffe.io.load_image(img)
net.blobs['data'].data[...] = transformer.preprocess('data',im)
out = net.forward()
labels = np.loadtxt(labels_filename, str, delimiter='\t')
prob= net.blobs['prob'].data[0].flatten()
print prob
order=prob.argsort()[:3]
return prob
# print 'the class is:',labels[order]
# f=file("/home/user/swfcode/caffe/label.txt","a+")
#f.write(labels[order]+'\n')
labels_filename = root +'result.txt'
for i in range(0, len(filelist)):
img= filelist[i]
prob=Test(img ,net)
np.argmax(prob)
mylist.append(np.argmax(prob))
file=open('result.txt','w')
file.write(str(mylist))
file.close()