小技巧（5）：将TT100K数据集转成VOC格式，并且用Python脚本选出45类超过100张的图片和XML

上一篇：小技巧（4）：将txt中的某两列数据写入csv文件中，制作图像分类标签

一、相关准备

1.1 下载数据集

http://cg.cs.tsinghua.edu.cn/traffic-sign/data_model_code/data.zip
TT100K数据集下载下来大概19.2G，解压后的文件夹如图

1.2 下载代码文件

https://github.com/cqfdch/TT100K_to_VOC

1.3 将相关文件移入代码文件

讲TT100K中的train文件、test文件夹、annotations.json文件移入代码文件夹。

二、创建标准的VOC文件夹

import os

# 建立相关文件夹
# build voc2007 folder structure
def make_voc_dir():
    root_dir = os.getcwd()
    os.makedirs(root_dir+'/VOC2007')
    os.makedirs("VOC2007"+'/Annotations')
    os.makedirs("VOC2007" + '/JPEGImages/ ')
    os.makedirs("VOC2007"+'/ImageSets')
    os.makedirs("VOC2007"+'/ImageSets/Main')

if __name__ == '__main__':
    make_voc_dir()

三、生成整个数据集的XML文件

import os
import json
from lxml import etree as ET
from xml.dom import minidom
#找出训练集和测试集中的不在45类的标注图片的id
def edit_xml(objects, id, dir):
    save_xml_path = os.path.join(dir, "%s.xml" % id)  # xml

    root = ET.Element("annotation")
    # root.set("version", "1.0")  
    folder = ET.SubElement(root, "folder")
    folder.text = "none"
    filename = ET.SubElement(root, "filename")
    filename.text = id + ".jpg"
    source = ET.SubElement(root, "source")
    source.text = "none"
    owner = ET.SubElement(root, "owner")
    owner.text = "halftop"
    size = ET.SubElement(root, "size")
    width = ET.SubElement(size, "width")
    width.text = str(2048)
    height = ET.SubElement(size, "height")
    height.text = str(2048)
    depth = ET.SubElement(size, "depth")
    depth.text = "3"
    segmented = ET.SubElement(root, "segmented")
    segmented.text = "0"
    for obj in objects:  #  
        object = ET.SubElement(root, "object")
        name = ET.SubElement(object, "name")  # number
        name.text = obj["category"]

        # meaning = ET.SubElement(object, "meaning")  # name
        # meaning.text = inf_value[0]
        pose = ET.SubElement(object, "pose")
        pose.text = "Unspecified"
        truncated = ET.SubElement(object, "truncated")
        truncated.text = "0"
        difficult = ET.SubElement(object, "difficult")
        difficult.text = "0"
        bndbox = ET.SubElement(object, "bndbox")
        xmin = ET.SubElement(bndbox, "xmin")
        xmin.text = str(int(obj["bbox"]["xmin"]))
        ymin = ET.SubElement(bndbox, "ymin")
        ymin.text = str(int(obj["bbox"]["ymin"]))
        xmax = ET.SubElement(bndbox, "xmax")
        xmax.text = str(int(obj["bbox"]["xmax"]))
        ymax = ET.SubElement(bndbox, "ymax")
        ymax.text = str(int(obj["bbox"]["ymax"]))
    tree = ET.ElementTree(root)
    tree.write(save_xml_path, encoding="UTF-8", xml_declaration=True)
    root = ET.parse(save_xml_path) 
    file_lines = minidom.parseString(ET.tostring(root, encoding="Utf-8")).toprettyxml(
        indent="\t") 
    file_line = open(save_xml_path, "w", encoding="utf-8")  
    file_line.write(file_lines)
    file_line.close()
    
def  getDirId(dir):  # get the  id list  of id.png
    names = os.listdir(dir)
    ids = []
    for name in names:
        # path = os.path.join(dir, name)
        # img  = cv2.imread(path)
        # w, h, c = img.shape
        # if name.endswith(".jpg") or name.endswith(".png"):
            # ids["%s" % name.split(".")[0]] = [w, h, c]
        ids.append(name.split(".")[0])
    return ids

def is_tt45(objects):
    flag = True
    json_file = open('./TT100K_VOC_classes.json', 'r')
    results = json.load(json_file)
    for obj in objects:
        text = obj["category"]
        for key in results.keys():
            flag1 = False
            if key == text:
                flag1 =True
                break
        if flag1 == False:
            flag = False
            break
    return flag
filedir = "annotations.json"
annos = json.loads(open(filedir).read())

trainIds = getDirId("train/")
testIds = getDirId("test/")

ids = annos["imgs"].keys() #  all img ids in .json 

dir_train = "xmlLabel1/train"
dir_test = "xmlLabel1/test"
if not os.path.exists(dir_train):
    os.makedirs(dir_train)
if not os.path.exists(dir_test):
    os.makedirs(dir_test)

Not_TT45_list_train = []
Not_TT45_list_val = []
for id in ids:
#  json 中的ID图片有待检测目标，且该id图片在 train文件夹中
    if len(annos["imgs"][id]["objects"]) > 0 and (id in trainIds):
        objects = annos["imgs"][id]["objects"]
        flag = is_tt45(objects)
        if flag is False:
            Not_TT45_list_train.append(id+'\n')
        edit_xml(objects, id, dir=dir_train)
        
    elif len(annos["imgs"][id]["objects"]) > 0 and (id in testIds):
        objects = annos["imgs"][id]["objects"]
        flag = is_tt45(objects)
        if flag is False:
            Not_TT45_list_val.append(id+'\n')
        edit_xml(objects, id, dir=dir_test)
with open("Not_TT45_list_train.txt" ,"a") as f:
    f.writelines(Not_TT45_list_train)
with open("Not_TT45_list_val.txt" ,"a") as f:
    f.writelines(Not_TT45_list_val)
    

四、删除txt中id对应XML和图片

import os
import glob

# 删除txt中id对应XML和图片
def delete_train_jpg(train_txt):
    root_dir = os.getcwd()
    for line in open(train_txt ,"r"):
        file_id = line.strip()
        # print(file_id)
        file_path = os.path.join(root_dir,"train",file_id+'.jpg')
        # print(file_path)
        os.remove(file_path)


def delete_test_jpg(test_txt):
    root_dir = os.getcwd()
    for line in open(test_txt ,"r"):
        file_id = line.strip()
        # print(file_id)
        file_path = os.path.join(root_dir,"test",file_id+'.jpg')
        # print(file_path)
        os.remove(file_path)

def delete_train_xml(train_txt):
    root_dir = os.getcwd()
    root_path = os.path.join(root_dir,"xmlLabel1")
    for line in open(train_txt,"r"):
        file_id = line.strip()
        # print(file_id)
        file_path = os.path.join(root_path,"train",file_id+'.xml')
        # print(file_path)
        os.remove(file_path)

def delete_test_xml(test_txt):
    root_dir = os.getcwd()
    root_path = os.path.join(root_dir,"xmlLabel1")
    for line in open(test_txt,"r"):
        file_id = line.strip()
        # print(file_id)
        file_path = os.path.join(root_path,"test",file_id+'.xml')
        # print(file_path)
        os.remove(file_path)
if __name__ == '__main__':
    train_txt = "Not_TT45_list_train.txt"
    test_txt = "Not_TT45_list_val.txt"
    delete_train_jpg(train_txt)
    delete_train_xml(train_txt)
    delete_test_jpg(test_txt)
    delete_test_xml(test_txt)

五 、生成tain.txt和val.txt文件

第一步，生成train.txt
需要将4_spilt_data.py中的下面两行改成

files_path = "/home/ch/Object_Detection/dataset/TT100K_to_VOC/xmlLabel1/train"
val_rate = 0

然后运行4_spilt_data.py文件，可得到train.txt和val.txt文件，其中val.txt文件是空文件。将train.txt文件移入VOC2017/ImageSets/Main中，删除val.txt文件。

第二步、生成val.txt
需要将4_spilt_data.py中的下面两行改成

	files_path = "/home/ch/Object_Detection/dataset/TT100K_to_VOC/xmlLabel1/test"
	val_rate =1

然后运行4_spilt_data.py文件，可得到train.txt和val.txt文件，其中train.txt文件是空文件。将val.txt文件移入VOC2017/ImageSets/Main中，删除train.txt文件。

import os
import random


files_path = "/home/ch/Object_Detection/dataset/TT100K_YOLO_Label-master/xmlLabel1/train"
if not os.path.exists(files_path):
    print("文件夹不存在")
    exit(1)
val_rate = 0

files_name = sorted([file.split(".")[0] for file in os.listdir(files_path)])
files_num = len(files_name)
val_index = random.sample(range(0, files_num), k=int(files_num*val_rate))
train_files = []
val_files = []
for index, file_name in enumerate(files_name):
    if index in val_index:
        val_files.append(file_name)
    else:
        train_files.append(file_name)

try:
    train_f = open("train.txt", "x")
    eval_f = open("val.txt", "x")
    train_f.write("\n".join(train_files))
    eval_f.write("\n".join(val_files))
except FileExistsError as e:
    print(e)
    exit(1)

最后将生成的train.txt和val.txt放进VOC2017/ImageSets/Main文件夹中

下一篇：小技巧（6）：进行BelgiumTSC交通标志数据集识别（定义自己的数据集）