pytorch训练剪枝版yoloV3遇见的几个坑

pytorch框架下的剪枝版本yoloV3下载地址

坑一：首先配置环境，参照主目录下的requirements.txt

# pip3 install -U -r requirements.txt
numpy
opencv-python
torch >= 1.2
matplotlib
pycocotools
tqdm
tb-nightly
future
Pillow

# Equivalent conda commands ----------------------------------------------------
# conda update -n base -c defaults conda
# conda install -yc anaconda future numpy opencv matplotlib tqdm pillow
# conda install -yc conda-forge scikit-image tensorboard pycocotools
# conda install -yc spyder-ide spyder-line-profiler
# conda install -yc pytorch pytorch torchvision

我这里是用的Anaconda环境，所以采用的第二种方法，但是由于pytorch官方的conda源对于网络不好的地方经常断掉，所以需要更换国内的conda源，搜过好多介绍这方面的，但pytorch下载依然很慢，由于国内的博客写的也是乱七八糟，pytorch正确的国内conda源（清华源）添加正确方式如下：

conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/
conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/
conda config --set show_channel_urls yes
conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/

然后再运行上面的步骤就会很快安装成功了。

坑二：

运行训练程序：

python3 train.py --data data/oxfordhand.data --batch-size 32 --accumulate 1 --weights weights/yolov3.weights --cfg cfg/yolov3-hand.cfg

训练自己的数据时报错：

多半是自己的数据路径以及数据格式不对，特别是train.txt需要检查，对照oxford_hand数据集自己比对，这个需要自行核实

坑三：

出现这个问题绝壁不是网上说的什么同一个batch内的图像大小要一致。因为这个都不是问题，谁家的框架不事先把这部分工作给做了。哪还需要我们自己动手。

出现这个的问题在于label文件有空文件，原因在于labelImg标注的时候xml文件有编辑过，之前标的对象后来被删除了，但XML文件还在，所以导致转换的时候里面的box是空的，转换后的label也就是空文件，导致utils/datasets.py中的第590行代码附近读取异常，故报错。解决办法就是把这部分图像以及xml文件删掉还有label文件也删掉。

最后附几个代码，自己写的：

copyjpg_maketxt.py（拷贝已有数据并生成该框架下可用的train.txt以及valid.txt)：

import os
import random
import shutil

trainval_percent = 0.1
xmlfilepath = '/home/user07/MyProjects/yolo/keras-yolo3-master/VOCdevkit/VOC2007/Annotations'
total_xml = os.listdir(xmlfilepath)

num = len(total_xml)
list = range(num)
tv = int(num * trainval_percent)
trainval = random.sample(list, tv)

if not os.path.exists('nightInstense'):
    os.makedirs('nightInstense')
if not os.path.exists('nightInstense/images/train'):
    os.makedirs('nightInstense/images/train')
if not os.path.exists('nightInstense/images/test'):
    os.makedirs('nightInstense/images/test')
if not os.path.exists('xml/test'):
    os.makedirs('xml/test')
if not os.path.exists('xml/train'):
    os.makedirs('xml/train')

ftrain = open('nightInstense/train.txt', 'w')
fval = open('nightInstense/valid.txt', 'w')

for i in list:
    jpgName='/home/user07/MyProjects/yolo/keras-yolo3-master/VOCdevkit/VOC2007/JPEGImages/'+total_xml[i][:-4]+'.jpg'
    xmlName='/home/user07/MyProjects/yolo/keras-yolo3-master/VOCdevkit/VOC2007/Annotations/'+total_xml[i]
    if i in trainval:
        name = '/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/images/test/'+total_xml[i][:-4]+'.jpg' + '\n'
        shutil.copyfile(jpgName,'nightInstense/images/test/'+total_xml[i][:-4]+'.jpg')
        shutil.copyfile(xmlName,'xml/test/'+total_xml[i])
        fval.write(name)
    else:
        name = '/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/images/train/'+total_xml[i][:-4]+'.jpg' + '\n'
        shutil.copyfile(jpgName,'nightInstense/images/train/'+total_xml[i][:-4]+'.jpg')
        shutil.copyfile(xmlName,'xml/train/'+total_xml[i])
        ftrain.write(name)

ftrain.close()
fval.close()

改写的voc_label.py(用于生成该框架下的lable文件)：

import xml.etree.ElementTree as ET
import pickle
import os
from os import listdir, getcwd
from os.path import join

classes = ["person", "train"]

def convert(size, box):
    dw = 1./size[0]
    dh = 1./size[1]
    x = (box[0] + box[1])/2.0
    y = (box[2] + box[3])/2.0
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x*dw
    w = w*dw
    y = y*dh
    h = h*dh
    return (x,y,w,h)

def convert_annotation(ldir, image_id):
    in_file = open('xml/%s/%s.xml'%(ldir, image_id))
    out_file = open('nightInstense/labels/%s/%s.txt'%(ldir, image_id), 'w')
    tree=ET.parse(in_file)
    root = tree.getroot()
    size = root.find('size')
    w = int(size.find('width').text)
    h = int(size.find('height').text)

    for obj in root.iter('object'):
        difficult = obj.find('difficult').text
        cls = obj.find('name').text
        if cls not in classes or int(difficult) == 1:
            continue
        cls_id = classes.index(cls)
        xmlbox = obj.find('bndbox')
        b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))
        bb = convert((w,h), b)
        out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')

wd = getcwd()

if not os.path.exists('nightInstense/labels/train'):
    os.makedirs('nightInstense/labels/train')
if not os.path.exists('nightInstense/labels/test'):
    os.makedirs('nightInstense/labels/test')


train_image_ids = open('nightInstense/train.txt').read().strip().split()
test_image_ids=open('nightInstense/valid.txt').read().strip().split()
for image_id in train_image_ids:
    convert_annotation('train', os.path.splitext(image_id.split('/')[-1])[0])
for image_id in test_image_ids:
    convert_annotation('test', os.path.splitext(image_id.split('/')[-1])[0])

最后附上一个剔除空label（remove_blank_labels.py）的代码：

import os


# llist=os.listdir('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/labels/train')

# for filename in llist:
#     if os.path.getsize(os.path.join('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/labels/train',filename))==0:
#         os.remove(os.path.join('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/labels/train',filename))
#         os.remove(os.path.join('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/images/train',os.path.splitext(filename)[0]+'.jpg'))
#         ftrain=open('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/train.txt','r')
#         lines=ftrain.readlines()
#         ftrain.close()
#         with open('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/train.txt','w') as w:
#             for l in lines:
#                 if str(os.path.splitext(filename)[0]+'.jpg') not in l:                              
#                     w.write(l)
#         w.close()
       
llist=os.listdir('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/images/train')
ftrain=open('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/train.txt','w')
for filename in llist:
    ftrain.write('/home/user07/MyProjects/YOLOv3-complete-pruning-master/data/nightInstense/images/train/'+filename+'\n')
ftrain.close()