足球视频位置与平面坐标的转换

依赖： C# OpenCVSharp WPF Numpy

目的：解决足球场上，人物投影到二维平面的位置坐标

图A / B/ C

一、基础概念

1.1标准球场的定义：

参考：https://zh.m.wikipedia.org/zh/%E8%B6%B3%E7%90%83%E5%A0%B4

图 D

1.2 配准思路

图A->图B，建立坐标关系对；

图B->图D，建立真实坐标系的配准基准点对（需要4个以上）

图D->图C，建立显示坐标系的配准基准点对（图C尺寸已知，所有关键位置A-J均已知）

对于全局，利用OpenCV FindHomography，求得D-C坐标转换矩阵T；

对于每帧图片，利用OpenCV FindHomography，求得B-D坐标转换矩阵T1；

将对象检测的球员投影到图3

二、代码实现

2.1 标画关键点

WPF界面

    <Grid>
        <Grid.ColumnDefinitions>
            <ColumnDefinition Width="0.4*"/>
            <ColumnDefinition Width="0.6*"/>
        Grid.ColumnDefinitions>
        <Grid x:Name="gridPointSelector" Width="350" Height="234" VerticalAlignment="Center" HorizontalAlignment="Center" >
            <Grid.ColumnDefinitions>
                <ColumnDefinition Width="0.15619*"/>
                <ColumnDefinition Width="0.34762*"/>
                <ColumnDefinition Width="0.34762*"/>
                <ColumnDefinition Width="0.15619*"/>
            Grid.ColumnDefinitions>
            <Grid.RowDefinitions>
                <RowDefinition Height="0.20441*" />
                <RowDefinition Height="0.59118*"/>
                <RowDefinition Height="0.20441*"/>
            Grid.RowDefinitions>
            <Image Source="{StaticResource 2D_field}" Grid.ColumnSpan="4" Grid.RowSpan="3" />
            <ToggleButton Content="A" Grid.Row="0" Grid.Column="0"  
                          HorizontalAlignment="Left" VerticalAlignment="Top"  Margin="-15,-15,0,0" Click="ToggleButton_Click"/>
            <ToggleButton Content="B" Grid.Row="0" Grid.Column="1" 
                          HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0"  Click="ToggleButton_Click"/>
            <ToggleButton Content="C" Grid.Row="0" Grid.Column="3" 
                          HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0"  Click="ToggleButton_Click"/>
            <ToggleButton Content="D" Grid.Row="1" Grid.Column="0" 
                          HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0"  Click="ToggleButton_Click"/>
            <ToggleButton Content="E" Grid.Row="1" Grid.Column="2" 
                          HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0"  Click="ToggleButton_Click"/>
            <ToggleButton Content="F" Grid.Row="2" Grid.Column="0" 
                          HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0"  Click="ToggleButton_Click"/>
            <ToggleButton Content="G" Grid.Row="2" Grid.Column="2" 
                          HorizontalAlignment="Right" VerticalAlignment="Top"   Click="ToggleButton_Click"/>
            <ToggleButton Content="H" Grid.Row="2" Grid.Column="0"
                          HorizontalAlignment="Left" VerticalAlignment="Bottom"    Click="ToggleButton_Click" />
            <ToggleButton Content="I" Grid.Row="2" Grid.Column="1" 
                          HorizontalAlignment="Right" VerticalAlignment="Bottom"  Click="ToggleButton_Click"/>
            <ToggleButton Content="J" Grid.Row="2" Grid.Column="3" 
                          HorizontalAlignment="Right" VerticalAlignment="Bottom"  Click="ToggleButton_Click"/>
        Grid>
        <Grid Grid.Column="1" x:Name="gridMain" ClipToBounds="True">
            <Image x:Name="imgMain" Source="{Binding OriginalImage }" Stretch="Uniform"/>
            <Canvas x:Name="canvas" Height="{Binding ElementName=imgMain,Path=ActualHeight}"
                    Width="{Binding ElementName=imgMain,Path=ActualWidth}" Background="Transparent"
                    IsEnabled="{Binding IsSelectting}" MouseLeftButtonUp="canvas_MouseLeftButtonUp">
            Canvas>
        Grid>
    Grid>

数据结构

public class VedioPointMark
{
        public string? Name { get; set; }
        /// 

        /// 球场坐标
        /// 
        public Point2d FiledPoint { get; set; }
        /// 

        /// 视频像素坐标
        /// 
        public Point2d VideoPixelPoint { get; set; }
    }

WPF后端代码

 public partial class UCPointMark : UserControl
    {
        MPointMark model;
        private readonly int ellipseSize = 16;
        ToggleButton? selectButton;
        public UCPointMark()
        {
            InitializeComponent();
            model = new MPointMark();
            this.DataContext = model;
        }

        private Border? DrawPoint()
        {
            if (selectButton == null)
                return null;
            Border myBorder = new Border();
            myBorder.CornerRadius = new CornerRadius(ellipseSize);
            myBorder.Width = ellipseSize;
            myBorder.BorderBrush = new SolidColorBrush(Colors.Blue);
            myBorder.Background = new SolidColorBrush(Colors.Red);
            myBorder.Child = new TextBlock() { 
                Text = selectButton.Content.ToString(),
                Foreground = new SolidColorBrush(Colors.White),
                TextAlignment = TextAlignment.Center};
            selectButton.Tag = myBorder;
            return myBorder;
        }

        public List<VedioPointMark> CalculateTransform() 
        {
            var markPoints = new List<VedioPointMark>();
            //必须选择4各以上
            if (canvas.Children.Count < 4)
                return markPoints;
            var allPoints = VideoFieldTransform.CreatePointMarks();
            var axisX = (double)model.OriginalImageSize.Width / this.imgMain.ActualWidth;
            var axisY = (double)model.OriginalImageSize.Height / this.imgMain.ActualHeight;
            foreach (var element in this.gridPointSelector.Children)
            {
                if (element is ToggleButton toggle && toggle.Tag is Border pixcelBorder
                    && pixcelBorder.Tag is Point pixcelPoint && toggle.IsChecked == true)
                {
                    var mark = allPoints.Where(p => p.Name?.Equals(toggle.Content.ToString()) == true).FirstOrDefault();
                    if (mark == null)
                        continue;
                    mark.VideoPixelPoint = new OpenCvSharp.Point2d( pixcelPoint.X * axisX, pixcelPoint.Y * axisY -40 );
                    markPoints.Add(mark);
                }
            }
            return markPoints;
        }

        private void canvas_MouseLeftButtonUp(object sender, MouseButtonEventArgs e)
        {
            try
            {
                var clickPoint = e.GetPosition(this.canvas);
                var ellipse = DrawPoint();
                if (null == ellipse) return;
                this.canvas.Children.Add(ellipse);
                Canvas.SetLeft(ellipse, clickPoint.X - ellipseSize / 2);
                Canvas.SetTop(ellipse, clickPoint.Y - ellipseSize / 2);
                ellipse.Tag = new Point(clickPoint.X, clickPoint.Y);
                model.IsSelectting = false;
            }
            catch (Exception ex)
            {
                this.Log(ex);
            }
        }

2.2 配准转换类

    public class VideoFieldTransform
    {
       private readonly Point2d[] point1;  //视频可视范围内的关键点坐标（4个以上）
       private readonly Point2d[] point2;  //真实球场坐标系 或 平面坐标系
       private readonly Mat H; //变换矩阵

        public VideoFieldTransform(List<VedioPointMark> pointMarks,bool haspoint2 = false)
        {
            if (!haspoint2)
            {
                //视频坐标投影到物理坐标
                pointMarks = pointMarks.OrderBy(p => p.Name).ToList();
                point1 = pointMarks.Select(p => p.FiledPoint).ToArray();
                point2 = pointMarks.Select(p => p.VideoPixelPoint).ToArray();
            }
            else
            {
                //物理坐标投影到图片
                point1 = new Point2d[] {
                                 new Point2d(0, 0),
                                  new Point2d(16.4f, 13.9f),
                                  new Point2d(52.5f, 0.0f),
                                  new Point2d(88.6f, 13.9f ),
                                  new Point2d(105f, 0.0f),
                                  new Point2d(105f, 68.0f),
                                  new Point2d(52.5f, 68f),
                                  new Point2d(0, 68f)
                              };
                point2 = new Point2d[] {
                                  new Point2d(0.0f, 0.0f ),
                                  new Point2d(164f, 152f ),
                                  new Point2d( 525f, 0.0f),
                                  new Point2d(886f, 152f),
                                  new Point2d(1050f, 0f),
                                  new Point2d(1050f, 699f),
                                   new Point2d(525f, 699f),
                                    new Point2d(0f, 699f),
                              };
            }
            H = CalculateHomoGraphy(haspoint2);
        }


        public static List<VedioPointMark> CreatePointMarks()
        {
            return new List<VedioPointMark>()
            {
                new VedioPointMark() {  Name = "A", FiledPoint = new Point2d(0, 0)},
                new VedioPointMark() {  Name = "D", FiledPoint = new Point2d(16.4f, 13.9f)},
                new VedioPointMark() {  Name = "B", FiledPoint = new Point2d(52.5f, 0.0f)},
                new VedioPointMark() {  Name = "E", FiledPoint = new Point2d(88.6f, 13.9f)},
                new VedioPointMark() {  Name = "C", FiledPoint = new Point2d(105f, 0.0f)},
                new VedioPointMark() {  Name = "J", FiledPoint = new Point2d(105f, 68.0f)},
                new VedioPointMark() {  Name = "I", FiledPoint = new Point2d(52.5f, 68f)},
                new VedioPointMark() {  Name = "H", FiledPoint = new Point2d(0, 68f)},
                new VedioPointMark() {  Name = "F", FiledPoint = new Point2d(16.4f, 40.2f)},
                new VedioPointMark() {  Name = "G", FiledPoint = new Point2d(88.6, 40.2f)},
             };
        }

        /// 

        ///     视频帧(假定摄像头位置不变)与 平面模式的球场位置的坐标系换算，求得矩阵
        /// 
        /// 
        /// 返回H 为变换矩阵
        private Mat CalculateHomoGraphy(bool make_rotate = false)
        {
            //var k = InputArray.Create(point2.GetData());
            if (!make_rotate)
                return Cv2.FindHomography(point2, point1, HomographyMethods.Ransac);
            return Cv2.FindHomography(point1,point2, HomographyMethods.Ransac);
        }

        /// 

        /// 根据坐标点(X,Y) 与 坐标系变换矩阵乘积，换算帧图像的位置到平面球场坐标的位置
        /// 
        public Point Transform(Point p)
        {
            var img2Bounds = new[]
            {
                new Point2d(p.X, p.Y)
            };
            var img2BoundsTransformed = Cv2.PerspectiveTransform(img2Bounds, H);
            var drawingPoints = img2BoundsTransformed.Select(p => (Point)p).FirstOrDefault();
            return drawingPoints;
        }

    }

2.3 初始化

图D->图C，建立显示坐标系的配准基准点对（图C尺寸已知，所有关键位置A-J均已知）

其中，pointMarks为WPF标记的点集

if (pointMarks?.Count > 3)
     proj_field_to_top = new VideoFieldTransform(pointMarks);  //真实球场坐标

从绝对球场尺寸到显示尺寸的变换，参见1.2 D-C变换

 proj_field_2d = new VideoFieldTransform(true);  //平面显示坐标

加载底图

//加载球场真实坐标系底图
D_field_photo = LoadImages.Load("2D_field.png");

初始化目标检测

//对象检测,此处略...参见AI机器学习（五）相关的内容 
detector = new DetectorYolov7();

2.4 逐帧变换

假设某个球员的Mat 区域，在视频中的像素位置标记为rect

//获取帧图片
foreach(var frame in LoadImages.LoadVideo("你的视频路径"))
{
    Mat imagedetect = new Mat();
    //投影所有的球员
    foreach(var prediction =detector.Detect(imagedetect))
    {
        var rect = prediction.Rectangle;
        //拷贝真实球场坐标底图  
        var J = D_field_photo.Clone();
        //转换坐标系
        var pfield = proj_field_to_top.Transform(new OpenCvSharp.Point(rect.X , rect.Y));
        //用于显示的图像
        var pshow = proj_field_2d.Transform(new OpenCvSharp.Point(pfield.X, pfield.Y));
        Cv2.Circle(J, pshow.X, pshow.Y, 10, Scalar.LightCyan, -1);
        //发送给前端WPF，这里没给出用于显示的页面，大家自己实现一个即可，只需要一个Image控件
        OnDisplay?.Invoke(null, J.Resize(new OpenCvSharp.Size(300, 200)).ToBytes());
    }
}