Qt在windows下利用onnxruntime调用Pytorch的torchvision中的mask-rcnn模型的实验记录

生成自定义模型：
博客：https://blog.csdn.net/u013685264/article/details/100564660
官方：https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
中文教程合集：https://www.w3cschool.cn/pytorch/pytorch-5yko3be6.html

lableme的批量处理
https://blog.csdn.net/weixin_43750626/article/details/112576938

@echo off
for %%i in (*.json) do labelme_json_to_dataset "%%i"
pause

onnx
https://github.com/onnx/onnx

onnx_runtime
https://github.com/microsoft/onnxruntime/

onnx量化
https://blog.csdn.net/znsoft/article/details/114637468
https://www.onnxruntime.ai/docs/how-to/quantization.html
经过测试，发现目前的torchvision的maskrcnn的模型可以量化，但是加载不了。加载时报错：
mobilenet_v2倒是可以。

Traceback (most recent call last):
  File "C:/Users/Administrator/Desktop/mask-rcnn/onnxTest.py", line 28, in <module>
    session = onnxruntime.InferenceSession("mask_rcnn_quant.onnx")
  File "D:\ProgramData\Anaconda3\envs\pytorch_cuda\lib\site-packages\onnxruntime\capi\onnxruntime_inference_collection.py", line 280, in __init__
    self._create_inference_session(providers, provider_options)
  File "D:\ProgramData\Anaconda3\envs\pytorch_cuda\lib\site-packages\onnxruntime\capi\onnxruntime_inference_collection.py", line 307, in _create_inference_session
    sess = C.InferenceSession(session_options, self._model_path, True, self._read_config_from_model)
onnxruntime.capi.onnxruntime_pybind11_state.Fail: [ONNXRuntimeError] : 1 : FAIL : Load model from mask_rcnn_quant.onnx failed:This is an invalid model. Error: two nodes with same node name (rpn.head.conv.bias_scale_node).

onnx c++
https://blog.csdn.net/baidu_34595620/article/details/112176278
https://github.com/microsoft/onnxruntime/blob/master/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp

 //*************************************************************************
  // initialize  enviroment...one enviroment per process
  // enviroment maintains thread pools and other state info
  Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");

  // initialize session options if needed
  Ort::SessionOptions session_options;
  session_options.SetIntraOpNumThreads(1);

  // 要用cuda的话，就注释下面这个。而且也要下载对应的onnxruntime库
  // 用cpu的话3.几秒，用gpu的话0.3秒。显卡yyds..
  // If onnxruntime.dll is built with CUDA enabled, we can uncomment out this line to use CUDA for this
  // session (we also need to include cuda_provider_factory.h above which defines it)
  // #include "cuda_provider_factory.h"
  // OrtSessionOptionsAppendExecutionProvider_CUDA(session_options, 1);

libtorch
https://www.jianshu.com/p/b0b6ab5002cd
qt libtorch
https://blog.csdn.net/gg864461719/article/details/113000235

默认保存torchvision的mask-rcnn的话，会导致在c++端调用报错，因此需要按照这个格式来保存（主要是因为动态尺寸的问题）：
https://github.com/pytorch/vision/issues/2251

#这样是不行的（官方给的例程）：
        >>> model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)
        >>> model.eval()
        >>> x = [torch.rand(3, 300, 400), torch.rand(3, 500, 400)]
        >>> predictions = model(x)
        >>>
        >>> # optionally, if you want to export the model to ONNX:
        >>> torch.onnx.export(model, x, "mask_rcnn.onnx", opset_version = 11)

#要这样才行
torch.onnx.export(model, [x.to(device)], "mask_rcnn.onnx",
                          input_names=['image'],
                          output_names=['boxes', 'labels', 'scores', 'masks'],
                          dynamic_axes={
     
                              'image': [1, 2],
                              "boxes": [0],
                              "labels": [0],
                              "scores": [0],
                              "masks": [0, 2, 3]},
                          opset_version=11)  # 11

参考：https://blog.csdn.net/baidu_34595620/article/details/112176278
参考：https://blog.csdn.net/u011341856/article/details/100041050
利用opencv+onnx_runtime可以这样子调用保存的onnx模型（注意，我这里的模型是只进行batch_size = 1 的物体的识别，假如需要同时进行多张图片的识别的话，需要有针对性地更改）。

void MainWindow::onnxInit()
{
     
    static Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");
    Ort::SessionOptions session_options;
    session_options.SetIntraOpNumThreads(1);

    session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);

#ifdef _WIN32
    const wchar_t* model_path = L"D:/MyDesktop/pytorchPrj/mask_rcnn.onnx";
#else
    const char* model_path = "mask_rcnn.onnx";
#endif

    mSession = new Ort::Session(env, model_path, session_options);
    // print model input layer (node names, types, shape etc.)
    Ort::AllocatorWithDefaultOptions allocator;

    // print number of model input nodes
    size_t num_input_nodes = mSession->GetInputCount();
    size_t num_output_nodes = mSession->GetOutputCount();

    for(int i = 0; i < num_input_nodes; i++)
    {
     
        QString inputName = mSession->GetInputName(i, allocator);
        auto inputType = mSession->GetInputTypeInfo(i);

        qDebug() << inputName << inputType.GetONNXType() << inputType.GetTensorTypeAndShapeInfo().GetShape();
    }

    for(int i = 0; i < num_output_nodes; i++)
    {
     
        QString outputName = mSession->GetOutputName(i, allocator);
        auto outputType = mSession->GetOutputTypeInfo(i);

        qDebug() << outputName << outputType.GetONNXType() << outputType.GetTensorTypeAndShapeInfo().GetShape();
    }
}

void MainWindow::onnxTest(cv::Mat &blob)
{
     
    std::vector<const char*> input_node_names = {
     "image"};
    std::vector<const char*> output_node_names = {
     "boxes", "labels", "scores", "masks"};

    //nchw
    int imgHeight = blob.size.p[2];
    int imgWidth = blob.size.p[3];

    std::vector<int64_t> input_node_dims = {
     3, imgHeight, imgWidth};
    size_t input_tensor_size = 3 * imgHeight * imgWidth ;

    // create input tensor object from data values
    auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
    Ort::Value input_tensor = Ort::Value::CreateTensor<float>(memory_info, (float*)blob.data, input_tensor_size, input_node_dims.data(), input_node_dims.size());
    assert(input_tensor.IsTensor());

    auto startTime = QDateTime::currentDateTime();
    qDebug() << "begin run:" << startTime;

    std::vector<Ort::Value> ort_inputs;
    ort_inputs.push_back(std::move(input_tensor));
    // score model & input tensor, get back output tensor
    auto output_tensors = mSession->Run(Ort::RunOptions{
     nullptr}, input_node_names.data(), ort_inputs.data(), ort_inputs.size(), output_node_names.data(), output_node_names.size());

    auto endTime = QDateTime::currentDateTime();

    qDebug() << "finish run:" << endTime;
    qDebug() << "interval:" << startTime.msecsTo(endTime) / 1000.0;



//    // Get pointer to output tensor float values
//    float* floatarr = output_tensors[0].GetTensorMutableData();
//    float* floatarr_mask = output_tensors[1].GetTensorMutableData();

    auto boxes = output_tensors[0].GetTypeInfo().GetTensorTypeAndShapeInfo().GetShape();
    auto labes = output_tensors[1].GetTypeInfo().GetTensorTypeAndShapeInfo().GetShape();

    qDebug() << "labels:" << boxes << labes;

    auto mask = output_tensors[3].GetTypeInfo().GetTensorTypeAndShapeInfo().GetShape();
    qDebug() << "masks:" << mask;

    Mat maskBlob;
    int blobSize[]={
     (int)mask[0], (int)mask[1], (int)mask[2], (int)mask[3]};
    float *dataPtr = output_tensors[3].GetTensorMutableData<float>();
    maskBlob = Mat(4, blobSize, CV_32F, dataPtr);

    std::vector<cv::Mat> maskImgList;
    cv::dnn::imagesFromBlob(maskBlob, maskImgList);


//    auto scoreData = output_tensors[2].GetTensorMutableData();
//    for(int i = 0; i < maskImgList.size(); i++)
//    {
     
//        float score = scoreData[i];

//        if(score < 0.9)
//        {
     
//            break;
//        }

//        QString windName = QString("mask_%1_%2").arg(i).arg(score);

//        imshow(windName.toStdString(), maskImgList[i]);
//    }

}


void MainWindow::on_pushButton_clicked()
{
     
    cv::Mat img = imread("D:/MyDesktop/pytorchPrj/glue/testImg/1.bmp", IMREAD_COLOR);

//    blobFromImage(frame, blob, 1.0, Size(frame.cols, frame.rows), Scalar(), true, false);
    auto blob = cv::dnn::blobFromImage(img, 1.0 / 255.0, Size(img.cols, img.rows), Scalar(), false, false);

    onnxTest(blob);
}