54、记录yolov7 训练、部署ncnn、部署mnn、部署rk3399 npu、部署oak vpu、部署openvino、部署TensorRT

基本思想：记录一下yolov7训练人头检测和部署oak的vpu相机上和rk3399 pro开发板上，完成需求

一、准备数据集，来自官方数据集，存在问题，已经修正

链接: https://pan.baidu.com/s/1SUGKRgxeV7ddZpLdILhOwA?pwd=atjt 提取码: atjt

二、在现有的数据集基础上转yolo数据集

ubuntu@ubuntu:~$ git clone https://github.com/WongKinYiu/yolov7.git
Cloning into 'yolov7'...
remote: Enumerating objects: 1094, done.
remote: Total 1094 (delta 0), reused 0 (delta 0), pack-reused 1094
Receiving objects: 100% (1094/1094), 69.89 MiB | 2.21 MiB/s, done.
Resolving deltas: 100% (517/517), done.

下载权重测试一下

ubuntu@ubuntu:~/yolov7/weights$
ubuntu@ubuntu:~/yolov7/weights$ wget https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7.pt
ubuntu@ubuntu:~/yolov7/weights$ cd ..

测试一下

ssh://[email protected]:22/usr/bin/python -u /home/ubuntu/yolov7/detect.py
Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', exist_ok=False, img_size=640, iou_thres=0.45, name='exp', no_trace=False, nosave=False, project='runs/detect', save_conf=False, save_txt=False, source='inference/images', update=False, view_img=False, weights='yolov7.pt')
YOLOR  v0.1-116-g8c0bf3f torch 1.9.0+cu111 CUDA:0 (NVIDIA GeForce RTX 3060, 12053.0MB)

Fusing layers... 
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
/home/ubuntu/.local/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
Model Summary: 306 layers, 36905341 parameters, 6652669 gradients, 104.5 GFLOPS
 Convert model to Traced-model... 
 traced_script_module saved! 
 model is traced! 

4 persons, 1 bus, 1 tie, Done. (11.6ms) Inference, (3.0ms) NMS
 The image with the result is saved in: runs/detect/exp2/bus.jpg
5 horses, Done. (10.8ms) Inference, (0.4ms) NMS
 The image with the result is saved in: runs/detect/exp2/horses.jpg
2 persons, 1 tie, 1 cake, Done. (12.3ms) Inference, (0.4ms) NMS
 The image with the result is saved in: runs/detect/exp2/image1.jpg
2 persons, 1 sports ball, Done. (10.9ms) Inference, (0.3ms) NMS
 The image with the result is saved in: runs/detect/exp2/image2.jpg
1 dog, 1 horse, Done. (10.7ms) Inference, (0.3ms) NMS
 The image with the result is saved in: runs/detect/exp2/image3.jpg
3 persons, 1 tie, Done. (9.6ms) Inference, (0.4ms) NMS
 The image with the result is saved in: runs/detect/exp2/zidane.jpg
Done. (0.335s)

Process finished with exit code 0

测试结果

三、转数据集

１)划分文件

# coding:utf-8

import os
import random
import argparse

parser = argparse.ArgumentParser()
# xml文件的地址，根据自己的数据进行修改 xml一般存放在Annotations下
parser.add_argument('--xml_path', default='/home/ubuntu/yolov7/trainData/Annotations', type=str, help='input xml label path')
# 数据集的划分，地址选择自己数据下的ImageSets/Main
parser.add_argument('--txt_path', default='/home/ubuntu/yolov7/trainData/ImageSets/Main', type=str, help='output txt label path')
opt = parser.parse_args()

trainval_percent = 1.0
train_percent = 0.9
xmlfilepath = opt.xml_path
txtsavepath = opt.txt_path
total_xml = os.listdir(xmlfilepath)
if not os.path.exists(txtsavepath):
    os.makedirs(txtsavepath)

num = len(total_xml)
list_index = range(num)
tv = int(num * trainval_percent)
tr = int(tv * train_percent)
trainval = random.sample(list_index, tv)
train = random.sample(trainval, tr)

file_train = open(txtsavepath + '/train.txt', 'w')
file_val = open(txtsavepath + '/val.txt', 'w')

for i in list_index:
    name = total_xml[i][:-4] + '\n'
    if i in trainval:
        if i in train:
            file_train.write(name)
        else:
            file_val.write(name)


file_train.close()
file_val.close()

2)生成标签

# -*- coding: utf-8 -*-
import xml.etree.ElementTree as ET
import os
from os import getcwd

sets = ['train', 'val']
classes = ["person"]  # 改成自己的类别voc2012


def convert(size, box):
    dw = 1. / (size[0])
    dh = 1. / (size[1])
    x = (box[0] + box[1]) / 2.0 - 1
    y = (box[2] + box[3]) / 2.0 - 1
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x * dw
    w = w * dw
    y = y * dh
    h = h * dh
    return x, y, w, h


def convert_annotation(image_id):
    in_file = open('/home/ubuntu/yolov7/trainData/Annotations/%s.xml' % (image_id))
    out_file = open('/home/ubuntu/yolov7/trainData/labels/%s.txt' % (image_id), 'w')
    tree = ET.parse(in_file)
    root = tree.getroot()
    size = root.find('size')
    w = int(size.find('width').text)
    h = int(size.find('height').text)
    for obj in root.iter('object'):
        difficult = None
        if obj.find('Difficult') is None:
            difficult = obj.find('difficult').text
        else:
            difficult = obj.find('Difficult').text
        cls = obj.find('name').text
        if cls not in classes or int(difficult) == 1:
            continue
        cls_id = classes.index(cls)
        xmlbox = obj.find('bndbox')
        b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
             float(xmlbox.find('ymax').text))
        b1, b2, b3, b4 = b
        # 标注越界修正
        if b2 > w:
            b2 = w
        if b4 > h:
            b4 = h
        b = (b1, b2, b3, b4)
        bb = convert((w, h), b)
        out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')


wd = getcwd()
for image_set in sets:
    if not os.path.exists('/home/ubuntu/yolov7/trainData/labels/'):
        os.makedirs('/home/ubuntu/yolov7/trainData/labels/')
    image_ids = open('/home/ubuntu/yolov7/trainData/ImageSets/Main/%s.txt' % (image_set)).read().strip().split()
    list_file = open('/home/ubuntu/yolov7/trainData/%s.txt' % (image_set), 'w')
    for image_id in image_ids:
        list_file.write('/home/ubuntu/yolov7/trainData/images/%s.jpg\n' % (image_id))
        convert_annotation(image_id)
    list_file.close()
print("voc_scrip finish")

四、修改coco.yaml

# COCO 2017 dataset http://cocodataset.org

# download command/URL (optional)
#download: bash ./scripts/get_coco.sh

# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/]
train: /home/ubuntu/yolov7/trainData/train.txt  # 118287 images
val: /home/ubuntu/yolov7/trainData/val.txt  # 5000 images
#test: ./coco/test-dev2017.txt  # 20288 of 40670 images, submit to https://competitions.codalab.org/competitions/20794

# number of classes
nc: 1

# class names
names: [ 'person']

 修改yolov7.yaml

nc: １  # number of classes

五、开始训练

ubuntu@ubuntu:~/yolov7$ python3 train.py --weights weights/yolov7.pt --cfg cfg/training/yolov7.yaml --data data/coco.yaml --device 0 --batch-size 8 --epoch 1000 --hyp data/hyp.scratch.p5.yaml

测试

ubuntu@ubuntu:~/yolov7$ python3 detect.py --weights runs/train/exp3/weights/best.pt 
Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', exist_ok=False, img_size=640, iou_thres=0.45, name='exp', no_trace=False, nosave=False, project='runs/detect', save_conf=False, save_txt=False, source='inference/images', update=False, view_img=False, weights=['runs/train/exp3/weights/best.pt'])
YOLOR  v0.1-116-g8c0bf3f torch 1.9.0+cu111 CUDA:0 (NVIDIA GeForce RTX 3050 Laptop GPU, 3910.5MB)

Fusing layers... 
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
IDetect.fuse
/home/ubuntu/.local/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
Model Summary: 314 layers, 36481772 parameters, 6194944 gradients, 103.2 GFLOPS
 Convert model to Traced-model... 
 traced_script_module saved! 
 model is traced! 

3 persons, Done. (22.3ms) Inference, (5.9ms) NMS
 The image with the result is saved in: runs/detect/exp3/bus.jpg
Done. (18.4ms) Inference, (0.2ms) NMS
 The image with the result is saved in: runs/detect/exp3/horses.jpg
4 persons, Done. (21.1ms) Inference, (0.6ms) NMS
 The image with the result is saved in: runs/detect/exp3/image1.jpg
2 persons, Done. (18.7ms) Inference, (0.6ms) NMS
 The image with the result is saved in: runs/detect/exp3/image2.jpg
2 persons, Done. (17.9ms) Inference, (0.6ms) NMS
 The image with the result is saved in: runs/detect/exp3/image3.jpg
2 persons, Done. (15.5ms) Inference, (0.6ms) NMS
 The image with the result is saved in: runs/detect/exp3/zidane.jpg
Done. (0.550s)

测试结果(发现尴尬的不，把马脸和狗脸识别成人头了。。。。)

六、转模型到onnx，不要把nms加

ubuntu@ubuntu:~/yolov7$ python3 export.py --weights runs/train/exp3/weights/best.pt --simplify --img-size 640 640 
Import onnx_graphsurgeon failure: No module named 'onnx_graphsurgeon'
Namespace(batch_size=1, conf_thres=0.25, device='cpu', dynamic=False, dynamic_batch=False, end2end=False, fp16=False, grid=False, img_size=[640, 640], include_nms=False, int8=False, iou_thres=0.45, max_wh=None, simplify=True, topk_all=100, weights='runs/train/exp3/weights/best.pt')
YOLOR  v0.1-116-g8c0bf3f torch 1.9.0+cu111 CPU

Fusing layers... 
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
RepConv.fuse_repvgg_block
IDetect.fuse
/home/ubuntu/.local/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
Model Summary: 314 layers, 36481772 parameters, 6194944 gradients, 103.2 GFLOPS

Starting TorchScript export with torch 1.9.0+cu111...
TorchScript export success, saved as runs/train/exp3/weights/best.torchscript.pt
CoreML export failure: No module named 'coremltools'

Starting TorchScript-Lite export with torch 1.9.0+cu111...
TorchScript-Lite export success, saved as runs/train/exp3/weights/best.torchscript.ptl

Starting ONNX export with onnx 1.12.0...

Starting to simplify ONNX...
ONNX export success, saved as runs/train/exp3/weights/best.onnx

Export complete (11.29s). Visualize with https://github.com/lutzroeder/netron.

七、转ncnn，直接调用ncnn官方的example即可

ubuntu@ubuntu:~/ncnn/build/install/bin$ ./onnx2ncnn /home/ubuntu/yolov7/runs/train/exp3/weights/best.onnx /home/ubuntu/yolov7/runs/train/exp3/weights/best.param /home/ubuntu/yolov7/runs/train/exp3/weights/best.bin

参考源码https://github.com/Tencent/ncnn/tree/master/examples

模型需要改掉后面的param文件这三个红框改成-1,否则会出现乱框

 cmakelists.txt

cmake_minimum_required(VERSION 3.16)
project(untitled22)
set(CMAKE_CXX_FLAGS "-std=c++11")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
set(CMAKE_CXX_STANDARD 11)
include_directories(${CMAKE_SOURCE_DIR})
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/include/ncnn)
find_package(OpenCV REQUIRED)
#message(STATUS ${OpenCV_INCLUDE_DIRS})
#添加头文件
include_directories(${OpenCV_INCLUDE_DIRS})
#链接Opencv库

add_library(libncnn STATIC IMPORTED)
set_target_properties(libncnn PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/lib/libncnn.a)


add_executable(untitled22 main.cpp)
target_link_libraries(untitled22 ${OpenCV_LIBS} libncnn )

main.cpp

// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#include "layer.h"
#include "net.h"

#if defined(USE_NCNN_SIMPLEOCV)
#include "simpleocv.h"
#else
#include 
#include 
#include 
#endif
#include 
#include 
#include 

#define MAX_STRIDE 32

struct Object
{
    cv::Rect_ rect;
    int label;
    float prob;
};

static inline float intersection_area(const Object& a, const Object& b)
{
    cv::Rect_ inter = a.rect & b.rect;
    return inter.area();
}

static void qsort_descent_inplace(std::vector