猫狗识别（tensorflow）kaggle
1.本人安装的环境为python3.7，pacharm1.1，windows系统
详细代码见这里
2.预处理
- 导入train data里面的图片
- 异常数据清理：训练集中大约包含了一些非猫或狗的图像，这些图片属于离群数据，可能会影响模型精度，需要移除。可以利用ImageNet预训练模型可以找出非猫或狗的图片以确定要清理哪些图片。鉴于时间原因且数据量不是太大，采用了人工挑选异常图片，这种方法并不推荐用于大数据集上，这些异常图片文件名如下：
cat.4688.jpg，cat.5418.jpg，cat.7377.jpg，cat.7564.jpg，cat.8100.jpg，cat.8456.jpg，cat.10029.jpg，cat.12272.jpg，dog.1259.jpg，dog.1895.jpg，dog.4367.jpg，dog.8736.jpg，dog.9517.jpg，dog.10190.jpg，dog.11299.jpg。 - .将第一步处理好的图片解码成为三维数组，然后将图片裁剪和补充进行标准化处理，转化为tensorflow能够识别的格式，分批次返回.
- pre_process.py
def get_files(file_dir):
cats = []
dogs = []
cats_label = []
dogs_label = []
img_dirs = os.listdir(file_dir)#读取文件名下所有！目录名(列表形式)
for img_name in img_dirs:# cat.0.jpg
name = img_name.split(".")# ['cat', '0', 'jpg']
if name[0] == "cat":
cats.append(file_dir + img_name)#此处不可以省为img_name,下个函数tf.train.slice_input_producer读取的是地址！！
cats_label.append(0)
else:
if name[0] == "dog":
dogs.append(file_dir + img_name)
dogs_label.append(1)
img_list = np.hstack((cats, dogs))#列表（字符串形式）
label_list = np.hstack((cats_label, dogs_label))#列表（整数形式）
return img_list, label_list
#############################################
def get_batch(image, label, image_w, image_h, batch_size, capacity):#capacity: 队列中 最多容纳图片的个数
input_queue = tf.train.slice_input_producer([image, label])#tf.train.slice_input_producer是一个tensor生成器，作用是
# 按照设定，每次从一个tensor列表中按顺序或者随机抽取出一个tensor放入文件名队列。
label = input_queue[1]
img_contents = tf.read_file(input_queue[0])#一维
image = tf.image.decode_jpeg(img_contents, channels=3)#解码成三维矩阵
image = tf.image.resize_image_with_crop_or_pad(image, image_w, image_h)
image = tf.cast(image, tf.float32)
image = tf.image.per_image_standardization(image)
# 生成批次 num_threads 有多少个线程根据电脑配置设置
image_batch, label_batch = tf.train.batch([image, label], batch_size=batch_size, num_threads=64, capacity=capacity)
return image_batch, label_batch
- 训练集划分验证集：通常模型训练时需要在验证集上观察评估指标是否达到要求，而本项目只提供了训练集和测试集，所以需要从训练集中划分一部分数据作为验证集，使用4:1的比例从训练集中随机挑选出一部分图片作为验证集。（可选）
- build_valiation_data.py
def build_valiation_data(src_dir, target_dir, validation_ratio):
if not os.path.exists(target_dir):
os.makedirs(target_dir)
files = os.listdir(src_dir)
total_size = len(files)
validation_size = int(total_size / validation_ratio)
print(validation_size)
random.shuffle(files)
for i in range(validation_size):
f = files[i]
shutil.move(os.path.join(src_dir, f), os.path.join(target_dir, f))#移动文件内容，src原地址，target目标地址
print("total size: {}, validation size: {}".format(total_size, validation_size))
build_valiation_data(src_dir=src_dir,target_dir=target_dir,validation_ratio=validation_ratio)
3.model.py
def inference(image, batch_size, n_classes):
with tf.variable_scope("conv1") as scope:#课本108，variable_scope控制get_variable是获取（reuse=True）还是创建变量
weights = tf.get_variable("weights", shape=[3,3,3,16], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32))
biases = tf.get_variable("biases", shape=[16], dtype=tf.float32,
initializer=tf.constant_initializer(0.1))
conv = tf.nn.conv2d(image, weights, strides=[1,1,1,1], padding="SAME")
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
with tf.variable_scope("pooling1_lrn") as scope:
pool1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding="SAME", name="pooling1")
norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001/9.0,beta=0.75, name="norm1")#局部响应归一化??????
with tf.variable_scope("conv2") as scope:
weights = tf.get_variable("weights", shape=[3,3,16,16], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32))
biases = tf.get_variable("biases", shape=[16], dtype=tf.float32,
initializer=tf.constant_initializer(0.1))
conv = tf.nn.conv2d(norm1, weights, strides=[1,1,1,1], padding="SAME")
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
with tf.variable_scope("pooling2_lrn") as scope:
norm2 = tf.nn.lrn(conv2, depth_radius=4, bias=1.0, alpha=0.001/9.0,beta=0.75, name="norm2")
pool2 = tf.nn.max_pool(norm2, ksize=[1,3,3,1], strides=[1,2,2,1], padding="SAME", name="pooling2")
with tf.variable_scope("local3") as scope:
reshape = tf.reshape(pool2, shape=[batch_size, -1])
dim = reshape.get_shape()[1].value
weights = tf.get_variable("weights", shape=[dim, 128], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))
biases = tf.get_variable("biases", shape=[128], dtype=tf.float32, initializer=tf.constant_initializer(0.1))
local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
with tf.variable_scope("local4") as scope:
weights = tf.get_variable("weights", shape=[128, 128], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))
biases = tf.get_variable("biases", shape=[128], dtype=tf.float32, initializer=tf.constant_initializer(0.1))
local4 = tf.nn.relu(tf.matmul(local3, weights) + biases,name="local4")
with tf.variable_scope("softmax_linear") as scope:
weights = tf.get_variable("weights", shape=[128, n_classes], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))
biases = tf.get_variable("biases", shape=[n_classes], dtype=tf.float32, initializer=tf.constant_initializer(0.1))
softmax_linear = tf.nn.relu(tf.matmul(local4, weights) + biases,name="softmax_linear")
return softmax_linear
def loss(logits, labels):#输出结果和标准答案
with tf.variable_scope("loss") as scope:
cross_entropy= tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels, name="entropy_per_example")
loss = tf.reduce_mean(cross_entropy)
tf.summary.scalar(scope.name +"/loss",loss)#对标量数据汇总和记录使用tf.summary.scalar
return loss
def training(loss, learning_rate):
with tf.name_scope("optimizer"):
global_step = tf.Variable(0, name="global_step", trainable=False)#定义训练的轮数，为不可训练的参数
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op= optimizer.minimize(loss, global_step=global_step)
#上两行等价于train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss,global_step=global_step)
return train_op
def evalution(logits, labels):
with tf.variable_scope("accuracy") as scope:
correct = tf.nn.in_top_k(logits, labels, 1)#下面
correct = tf.cast(correct, tf.float16)
accuracy = tf.reduce_mean(correct)
tf.summary.scalar(scope.name+"/accuracy", accuracy)#用来显示标量信息
return accuracy
4.train.py
def run_training():
train_dir = "D:\新建文件夹\python foot/train/"
log_train_dir = "D:\新建文件夹\python foot/train_savenet/"
vadiation_dir='D:\新建文件夹\python foot/valiation/'
train,train_labels = pre_process.get_files(train_dir)
train_batch, train_label_batch = pre_process.get_batch(train, train_labels, IMG_W,IMG_H,BATCH_SIZE,CAPACITY)
train_logits= model.inference(train_batch, BATCH_SIZE, N_CLASSES)
train_loss= model.loss(train_logits, train_label_batch)
train_op = model.training(train_loss, LEARNING_RATE)
train_acc = model.evalution(train_logits, train_label_batch)
summary_op = tf.summary.merge_all()#merge_all 可以将所有summary全部保存到磁盘，以便tensorboard显示。
# 一般这一句就可显示训练时的各种信息。
#vadiation, vadiation_labels = pre_process.get_files(vadiation_dir)
#vadiation_batch, vadiation_label_batch = pre_process.get_batch(vadiation, vadiation_labels, IMG_W,IMG_H,BATCH_SIZE, CAPACITY)
#vadiation_logits = model.inference(vadiation_batch, BATCH_SIZE, N_CLASSES)
#vadiation_loss = model.loss(vadiation_logits, vadiation_label_batch)
#vadiation_acc = model.evalution(vadiation_logits, vadiation_label_batch)
sess = tf.Session()
train_writer =tf.summary.FileWriter(log_train_dir, sess.graph)#指定一个文件用来保存图
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
# Coordinator 和 start_queue_runners 监控 queue 的状态，不停的入队出队
coord = tf.train.Coordinator()#https://blog.csdn.net/weixin_42052460/article/details/80714539
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
for step in np.arange(STEP):
if coord.should_stop():
break
_, tra_loss, tra_acc = sess.run([train_op, train_loss, train_acc])
if step % 50 == 0:#%.2f表示输出浮点数并保留两位小数。%%表示直接输出一个%
print("step %d, train loss = %.2f, train accuracy = %.2f%%" %(step, tra_loss, tra_acc*100.0))
summary_str = sess.run(summary_op)
train_writer.add_summary(summary_str, step) #?????????????
if step % 2000 == 0 or (step+1) ==STEP:
# 每隔2000步保存一下模型，模型保存在 checkpoint_path 中
print("step %d, vadiation loss = %.2f, vadiation accuracy = %.2f%%" % (step, vadiation_loss, vadiation_acc * 100.0))
checkpoint_path = os.path.join(log_train_dir, "model.ckpt")
saver.save(sess, checkpoint_path, global_step=step)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
run_training()
5.test.py
test = "D:\新建文件夹\python foot/test/"
file = os.listdir(test) # os.listdir()返回指定目录下的所有文件和目录名。
n = len(file)
df = pd.read_csv("D:\新建文件夹\python foot/sample_submission.csv")
for i in range(1,n):
img_dir = os.path.join(test, file[i]) # 判断是否存在文件或目录name
image = Image.open(img_dir)
image = image.resize([208, 208])
image = np.array(image)
test_array= image
#print(test_array.shape)
with tf.Graph().as_default():#https://www.cnblogs.com/studylyn/p/9105818.html
BATCH_SIZE = 1
N_CLASSES = 2
image = tf.cast(test_array, tf.float32)
image = tf.image.per_image_standardization(image)
image = tf.reshape(image,[1,208,208,3])
#test, train_labels = pre_process.get_files(test)
#image, _ = pre_process.get_batch(test, train_labels, IMG_W, IMG_H, BATCH_SIZE, CAPACITY)
logit = model.inference(image, BATCH_SIZE, N_CLASSES)
logit = tf.nn.softmax(logit)
x =tf.placeholder(tf.float32, shape =[208,208,3])
log_test_dir = "D:\新建文件夹\python foot/train_savenet"
saver = tf.train.Saver()
with tf.Session() as sess:
print("从指定路径中加载模型。。。")
#将模型加载到sess中
ckpt = tf.train.get_checkpoint_state(log_test_dir)
if ckpt and ckpt.model_checkpoint_path:#https://blog.csdn.net/u011500062/article/details/51728830/
global_step = ckpt.model_checkpoint_path.split("/")[-1].split("-")[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print("模型加载成功，训练的步数为 %s", global_step)
else:
print("模型加载失败，文件没有找到。")
#将图片输入到模型计算
prediction = sess.run(logit, feed_dict={x: test_array})
prediction=prediction.clip(min=0.005, max=0.995)
# 将图片输入到模型计算
#print(prediction[:, 1])
df.set_value(i-1, 'label', prediction[:, 1])
#print('猫的概率 %.6f' %prediction[:, 0])
#print('狗的概率 %.6f' %prediction[:, 1])
df.to_csv('D:\新建文件夹\python foot/pred.csv', index=None)
# 测试
evaluate_one_img()