火灾数据集处理
数据集
下载链接：https://github.com/steffensbola/furg-fire-dataset
内容：MP4的视频和对应的xml文件
问题：1.This XML was generated using OpenCv 2.4.9
2.xml文件中很多帧的图片没有坐标信息
目标：处理成VOC2007数据集格式
一，视频解帧
import cv2
import numpy as np
import os
def save_img():
video_path = r"F:\Fire\furg_fire_dataset\mp4/"
save_path = r"F:\Fire\furg_fire_dataset\img/"
videos = os.listdir(video_path)
print(len(videos),videos)
for video_name in videos:
file_name = video_name.split('.')[0]
# print(file_name)
folder_name = save_path + file_name
# print(folder_name)
# exit()
os.makedirs(folder_name, exist_ok=True)
print("="*10+'开始处理视频:%s'%video_name+"="*10)
vc = cv2.VideoCapture(video_path + video_name)
c = 0
rval = vc.isOpened()
while rval:
rval, frame = vc.read()
pic_path = folder_name + '/'
if rval:
cv2.imwrite(pic_path + str(c) + '.jpg', frame)
cv2.waitKey(1)
c += 1
else:
break
print('='*10+'%s图片数量：%i'%(video_name,c)+'='*10)
vc.release()
print('save_success')
print(folder_name)
save_img()
二，处理原始xml文件，使用库xml.etree.ElementTree和xml.dom.minidom解析我都没成功。发现里面的节点格式不对，我就换了种处理方式，把xml格式转成txt文本格式。
import os
import re
import re
path = r"F:\Fire\furg_fire_dataset\ann/"
save_path = r"F:\Fire\furg_fire_dataset\txt/"
xml_file = os.listdir(path)
for txt_file in xml_file:
print('='*10+'开始处理：'+txt_file+'='*10)
file = txt_file.split('.')[0]
# print(file)
new_file = file+'.txt'
xml_dir = path+txt_file
txt_dir = path+new_file
print(xml_dir,txt_dir)
# exit()
os.rename(xml_dir,txt_dir)
print('+'*10+'%s更改格式：'%txt_file+new_file+'+'*10)
print(save_path+txt_file)
# exit()
pic_name = []
box_data = {}
index = ['0,','1','2','3','4','5','6','7','8','9']
# with open(path+txt_file) as f:
with open(txt_dir) as f:
strs = f.readlines()
# print(strs)
count = 0
for i in strs:
# print(i)
# print(i.strip())
c = i.strip()[0]
# print('===============',c)
# print(type(c))
t_f = i.strip()[-18:]
im_file = i.strip()[-14:]
# print(im_file)
# print('t-f===================',t_f)
# exit()
if t_f == '':
box_data[count] = i.strip()
count +=1
# print('+++++++++++++++++++++++++++++++++++',i)
elif im_file == '':
pic_name.append(i.strip())
# print('===================================',i)
else:
print('出错%s'%i)
# print("pic_name:",pic_name)
# print("box_data:",box_data)
dict = {}
for i in range(len(pic_name)-1):
pic = pic_name[i].split('三，处理视频解帧的图片，删除没有标签信息的图片
import os
import re
import glob
img_path = r"F:\Fire\furg_fire_dataset\img/"
txt_path = r"F:\Fire\furg_fire_dataset\txt/"
file_names = os.listdir(img_path)
for file_name in file_names:
print('&'*10+'开始处理图片文件夹：'+file_name+'&'*10)
# exit()
img_file = []
txt_file = []
txt_dir = txt_path+file_name+'.txt'
print("="*10+'处理txt文本：'+txt_dir+'='*10)
with open(txt_dir) as f:
strs = f.readlines()
print('strs:',strs)
for txt in strs:
# print(txt)
s = txt.strip().split(' ')
# print(s,s[0],s[1])
if s[1]=='':
img_file.append(s[0])
else:
txt_file.append(txt.strip())
# print(txt_file)
# print(len(txt_file),len(strs))
print('错误图片:',img_file,len(img_file))
with open(txt_dir,'w') as k:
for t in txt_file:
k.write(t+'\n')
for im in os.listdir(os.path.join(img_path, file_name)):
# print(im)
# print(i)
pic = im.split('.')[0]
t_f = pic in img_file
# print(t_f)
if t_f:
print('+'*12+'错误图片数：'+str(len(img_file))+'+'*12)
print('+'*8+'删除图片：'+img_path +file_name+'/'+ im+'+'*8)
os.remove(img_path+file_name+'/'+im)
四，把图片重命名并复制到空的文件夹
import os
import shutil
# 复制图片到新的文件夹
path = r"F:\Fire\furg_fire_dataset\img/"
xml_path = r"E:\xunleidownload\fire-dataset-dunnings\videos\furg-fire-dataset\Annotations/"
save_path = r"F:\Fire\furg_fire_dataset\pic/"
file_path = os.listdir(path)
print(len(file_path),file_path)
for i in file_path:
img_files = os.listdir(path+i)
print(i)
for img in img_files:
# print(img)
src_dir = path+i+'/'+img
dst_dir = path+i+'/'+i+img
pic_dir = save_path+ i+img
# print(img)
print(src_dir)
print(dst_dir)
print(pic_dir)
# exit()
os.rename(src_dir,dst_dir)
shutil.copyfile(dst_dir,pic_dir)
五，使用txt的标签信息，生成图片对应的xml文件
import os, sys
import glob
from PIL import Image
obj_name = 'fire'
src_img_dir = r"F:\Fire\furg_fire_dataset\img\HouseTwo"
src_txt_dir = r"F:\Fire\furg_fire_dataset\txt\HouseTwo.txt"
# src_xml_dir = r"F:\Fire\furg_fire_dataset\xml"
src_xml_dir = r"F:\Fire\furg_fire_dataset\xml"
# src_txt_dir = src_img_dir + 'F:\Fire\furg_fire_dataset\xml'
# src_xml_dir = src_img_dir
img_Lists = glob.glob(src_img_dir + '/*.jpg')
print(img_Lists)
img_basenames = [] # e.g. 100.jpg
for item in img_Lists:
img_basenames.append(os.path.basename(item))
img_names = [] # e.g. 100
for item in img_basenames:
temp1, temp2 = os.path.splitext(item)
img_names.append(temp1)
for img in img_names:
im = Image.open((src_img_dir + '/' + img + '.jpg'))
print('img_name:',img)
img_label = img.split('seTwo')[1]
print('img_label:',img_label)
# exit()
width, height = im.size
# open the crospronding txt file
gt = open(src_txt_dir).read().splitlines() # int x，int y，int width，int height x,y是左上角
print('boxes:',gt)
# exit()
# gt = open(src_txt_dir + '/gt_' + img + '.txt').read().splitlines()
# write in xml file
# os.mknod(src_xml_dir + '/' + img + '.xml')
xml_file = open((src_xml_dir + '/' + img + '.xml'), 'w')
xml_file.write('\n')
xml_file.write(' VOC2007 \n')
xml_file.write(' ' + str(img) + '.jpg' + ' \n')
xml_file.write(' \n')
xml_file.write(' ' + str(width) + ' \n')
xml_file.write(' ' + str(height) + ' \n')
xml_file.write(' 3 \n')
xml_file.write(' \n')
# write the region of image on xml file
for img_each_label in gt:
spt = img_each_label.split(' ') # 这里如果txt里面是以逗号‘，’隔开的，那么就改为spt = img_each_label.split(',')。
# print("========",spt)
if spt[0] == img_label:
# print(spt[0])
# print(spt[-1])
# print(int(float(int(spt[1])+int(spt[3]))))
# print(int(spt[2])+int(spt[4]))
# exit()
xml_file.write(' \n')
xml_file.write(' ')
六，VOC2007格式的JPEGImages和Annotations已备齐。还缺ImageSets\Main下的训练测试文本。
import os
import random
from glob import glob
import shutil
saved_path = r'F:\Fire\smoke\MP4\smoke_detect-PascalVOC-export\VOC2007/'
# saved_path = r"E:\xunleidownload\VOCdevkit\VOC2007/"
# trainval_percent = 0.66
trainval_percent = 0.8
# train_percent = 0.8
#
train_percent = 0.8
xmlfilepath = saved_path + 'Annotations/'
txtsavepath = saved_path + 'ImageSets/Main'
total_xml = os.listdir(xmlfilepath)
num=len(total_xml)
list=range(num)
tv=int(num*trainval_percent)
tr=int(tv*train_percent)
trainval= random.sample(list,tv)
train=random.sample(trainval,tr)
ftrainval = open(saved_path + 'ImageSets/Main/trainval.txt', 'w')
ftest = open(saved_path + 'ImageSets/Main/test.txt', 'w')
ftrain = open(saved_path + 'ImageSets/Main/train.txt', 'w')
fval = open(saved_path + 'ImageSets/Main/val.txt', 'w')
for i in list:
name=total_xml[i][:-4]+'\n'
if i in trainval:
ftrainval.write(name)
if i in train:
ftrain.write(name)
else:
fval.write(name)
else:
ftest.write(name)
ftrainval.close()
ftrain.close()
fval.close()
ftest .close()