C# Onnx LSTR 基于Transformer的端到端实时车道线检测

目录

效果

模型信息

项目

代码

下载

---

效果

端到端实时车道线检测

模型信息

lstr_360x640.onnx

Inputs
-------------------------
name：input_rgb
tensor：Float[1, 3, 360, 640]
name：input_mask
tensor：Float[1, 1, 360, 640]
---------------------------------------------------------------

Outputs
-------------------------
name：pred_logits
tensor：Float[1, 7, 2]
name：pred_curves
tensor：Float[1, 7, 8]
name：foo_out_1
tensor：Float[1, 7, 2]
name：foo_out_2
tensor：Float[1, 7, 8]
name：weights
tensor：Float[1, 240, 240]
---------------------------------------------------------------

项目

VS2022+.net framework 4.8

OpenCvSharp 4.8

Microsoft.ML.OnnxRuntime 1.16.2

代码

创建输入tensor使用指针

float[] input_tensor_data = new float[1 * 3 * inpHeight * inpWidth];
for (int c = 0; c < 3; c++)
{
    for (int i = 0; i < row; i++)
    {
        for (int j = 0; j < col; j++)
        {
            float pix = ((byte*)(resize_image.Ptr(i).ToPointer()))[j * 3 + c];
            input_tensor_data[c * row * col + i * col + j] = (float)((pix / 255.0 - mean[c]) / std[c]);
        }
    }
}
input_tensor = new DenseTensor(input_tensor_data, new[] { 1, 3, inpHeight, inpWidth });

using Microsoft.ML.OnnxRuntime.Tensors;
using Microsoft.ML.OnnxRuntime;
using OpenCvSharp;
using System;
using System.Collections.Generic;
using System.Windows.Forms;
using System.Linq;
using System.IO;
using System.Text;
using System.Drawing;

namespace Onnx_Demo
{
    public partial class frmMain : Form
    {
        public frmMain()
        {
            InitializeComponent();
        }

        string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string image_path = "";

        DateTime dt1 = DateTime.Now;
        DateTime dt2 = DateTime.Now;

        int inpWidth;
        int inpHeight;

        Mat image;

        string model_path = "";

        float[] factors = new float[2];

        SessionOptions options;
        InferenceSession onnx_session;
        Tensor input_tensor;
        Tensor mask_tensor;
        List input_ontainer;

        IDisposableReadOnlyCollection result_infer;
        DisposableNamedOnnxValue[] results_onnxvalue;

        Tensor result_tensors;

        int len_log_space = 50;
        float[] log_space;

        float[] mean = new float[] { 0.485f, 0.456f, 0.406f };
        float[] std = new float[] { 0.229f, 0.224f, 0.225f };

        Scalar[] lane_colors = new Scalar[] { new Scalar(68, 65, 249), new Scalar(44, 114, 243), new Scalar(30, 150, 248), new Scalar(74, 132, 249), new Scalar(79, 199, 249), new Scalar(109, 190, 144), new Scalar(142, 144, 77), new Scalar(161, 125, 39) };

        private void button1_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            if (ofd.ShowDialog() != DialogResult.OK) return;

            pictureBox1.Image = null;
            pictureBox2.Image = null;
            textBox1.Text = "";

            image_path = ofd.FileName;
            pictureBox1.Image = new System.Drawing.Bitmap(image_path);
            image = new Mat(image_path);
        }

        private void Form1_Load(object sender, EventArgs e)
        {

            // 创建输入容器
            input_ontainer = new List();

            // 创建输出会话
            options = new SessionOptions();
            options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_INFO;
            options.AppendExecutionProvider_CPU(0);// 设置为CPU上运行

            // 创建推理模型类，读取本地模型文件
            model_path = "model/lstr_360x640.onnx";

            inpWidth = 640;
            inpHeight = 360;

            onnx_session = new InferenceSession(model_path, options);

            // 创建输入容器
            input_ontainer = new List();

            FileStream fileStream = new FileStream("model/log_space.bin", FileMode.Open);
            BinaryReader br = new BinaryReader(fileStream, Encoding.UTF8);

            log_space = new float[len_log_space];

            byte[] byteTemp;
            float fTemp;
            for (int i = 0; i < len_log_space; i++)
            {
                byteTemp = br.ReadBytes(4);
                fTemp = BitConverter.ToSingle(byteTemp, 0);
                log_space[i] = fTemp;
            }
            br.Close();

            image_path = "test_img/0.jpg";
            pictureBox1.Image = new Bitmap(image_path);

        }

        private unsafe void button2_Click(object sender, EventArgs e)
        {
            if (image_path == "")
            {
                return;
            }
            textBox1.Text = "检测中，请稍等……";
            pictureBox2.Image = null;
            System.Windows.Forms.Application.DoEvents();

            //图片缩放
            image = new Mat(image_path);

            int img_height = image.Rows;
            int img_width = image.Cols;

            Mat resize_image = new Mat();
            Cv2.Resize(image, resize_image, new OpenCvSharp.Size(inpWidth, inpHeight));

            int row = resize_image.Rows;
            int col = resize_image.Cols;

            float[] input_tensor_data = new float[1 * 3 * inpHeight * inpWidth];
            for (int c = 0; c < 3; c++)
            {
                for (int i = 0; i < row; i++)
                {
                    for (int j = 0; j < col; j++)
                    {
                        float pix = ((byte*)(resize_image.Ptr(i).ToPointer()))[j * 3 + c];
                        input_tensor_data[c * row * col + i * col + j] = (float)((pix / 255.0 - mean[c]) / std[c]);
                    }
                }
            }
            input_tensor = new DenseTensor(input_tensor_data, new[] { 1, 3, inpHeight, inpWidth });

            float[] input_mask_data = new float[1 * 1 * inpHeight * inpWidth];
            for (int i = 0; i < input_mask_data.Length; i++)
            {
                input_mask_data[i] = 0.0f;
            }
            mask_tensor = new DenseTensor(input_mask_data, new[] { 1, 1, inpHeight, inpWidth });

            //将 input_tensor 放入一个输入参数的容器，并指定名称
            input_ontainer.Add(NamedOnnxValue.CreateFromTensor("input_rgb", input_tensor));
            input_ontainer.Add(NamedOnnxValue.CreateFromTensor("input_mask", mask_tensor));

            dt1 = DateTime.Now;
            //运行 Inference 并获取结果
            result_infer = onnx_session.Run(input_ontainer);
            dt2 = DateTime.Now;

            //将输出结果转为DisposableNamedOnnxValue数组
            results_onnxvalue = result_infer.ToArray();

            float[] pred_logits = results_onnxvalue[0].AsTensor().ToArray();
            float[] pred_curves = results_onnxvalue[1].AsTensor().ToArray();

            int logits_h = results_onnxvalue[0].AsTensor().Dimensions[1];
            int logits_w = results_onnxvalue[0].AsTensor().Dimensions[2];
            int curves_w = results_onnxvalue[1].AsTensor().Dimensions[2];

            List good_detections = new List();
            List> lanes = new List>();
            for (int i = 0; i < logits_h; i++)
            {
                float max_logits = -10000;
                int max_id = -1;
                for (int j = 0; j < logits_w; j++)
                {
                    float data = pred_logits[i * logits_w + j];
                    if (data > max_logits)
                    {
                        max_logits = data;
                        max_id = j;
                    }
                }
                if (max_id == 1)
                {
                    good_detections.Add(i);
                    int index = i * curves_w;
                    List lane_points = new List();
                    for (int k = 0; k < len_log_space; k++)
                    {
                        float y = pred_curves[0 + index] + log_space[k] * (pred_curves[1 + index] - pred_curves[0 + index]);
                        float x = (float)(pred_curves[2 + index] / Math.Pow(y - pred_curves[3 + index], 2.0) + pred_curves[4 + index] / (y - pred_curves[3 + index]) + pred_curves[5 + index] + pred_curves[6 + index] * y - pred_curves[7 + index]);
                        lane_points.Add(new OpenCvSharp.Point(x * img_width, y * img_height));
                    }
                    lanes.Add(lane_points);
                }
            }

            Mat result_image = image.Clone();

            //draw lines
            List right_lane = new List();
            List left_lane = new List();
            for (int i = 0; i < good_detections.Count; i++)
            {
                if (good_detections[i] == 0)
                {
                    right_lane.Add(i);
                }
                if (good_detections[i] == 5)
                {
                    left_lane.Add(i);
                }
            }

            if (right_lane.Count() == left_lane.Count())
            {
                Mat lane_segment_img = result_image.Clone();

                List points = new List();

                points.AddRange(lanes.First());

                points.Reverse();

                points.AddRange(lanes[left_lane[0]]);

                Cv2.FillConvexPoly(lane_segment_img, points, new Scalar(0, 191, 255));
                Cv2.AddWeighted(result_image, 0.7, lane_segment_img, 0.3, 0, result_image);
            }

            for (int i = 0; i < lanes.Count(); i++)
            {
                for (int j = 0; j < lanes[i].Count(); j++)
                {
                    Cv2.Circle(result_image, lanes[i][j], 3, lane_colors[good_detections[i]], -1);
                }
            }

            pictureBox2.Image = new System.Drawing.Bitmap(result_image.ToMemoryStream());
            textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
        }

        private void pictureBox2_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox2.Image);
        }

        private void pictureBox1_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox1.Image);
        }
    }
}

下载

源码下载