[Unity3D——用代码说明一切]Unity结合Kinect2体感开发：Kinect控制U3D中的模型

实现效果：

Kinect1和Kinect2的关键骨骼位置信息：

Unity版本：5.3.5

项目包下载地址：点击打开链接

关键原理：

1.利用得到的Kinect的Body信息得到所有骨骼信息

2.通过骨骼信息得到对应点骨骼的旋转信息

3.将旋转信息用到本地的模型骨骼上

4.每个骨骼点的方向向量的计算

主要知识点：

要理解坐标系中Vector3和Quaternion之间的关系,世界坐标和相对坐标的相互转换

主要脚本：

KinectBodyControl用于获得Kinect获得的身体信息并创建模型

KinectBodyControlLogic用于控制模型骨骼的更新

CamerColorView用于UGUI显示Kinect摄像头内容

using UnityEngine;
using System.Collections;
using Windows.Kinect;
using System.Collections.Generic;

/// 
/// 体感游戏对象控制器
/// 
public class KinectBodyControl : MonoBehaviour
{
    //Kinect对象
    private KinectSensor _kinectSensor;
    //身体信息读取流
    private BodyFrameReader _bodyReader;

    //从Kinect得到的身体信息 每帧获取 
    private Body[] _bodyDatas = null;

    //场景中的体感游戏对象
    private Dictionary _bodies = new Dictionary();
    //当前追踪到的游戏对象
    private List _nowTrackedIDs = new List();
    //用于缓存已知的追踪到的游戏对象
    private List _nowCreatedBodyID = new List();

    void Start()
    {
        //得到kinect设备对象
        _kinectSensor = KinectSensor.GetDefault();

        if (_kinectSensor != null)
        {
            //得到身体数据流
            _bodyReader = _kinectSensor.BodyFrameSource.OpenReader();

            //设备是否开启
            if (!_kinectSensor.IsOpen)
                _kinectSensor.Open();
        }
    }

    void Update()
    {
        //通过Kinect得到身体信息
        GetBodyInfos();
        //得到身体信息后 更新场景中的身体对象
        UpdateBodyObject();
    }

    /// 
    /// 得到身体信息
    /// 
    private void GetBodyInfos()
    {
        if (_bodyReader != null)
        {
            BodyFrame frame = _bodyReader.AcquireLatestFrame();
            if (frame != null)
            {
                //初始化身体数组信息
                if (_bodyDatas == null)
                    _bodyDatas = new Body[_kinectSensor.BodyFrameSource.BodyCount];

                //得到该帧的身体信息
                frame.GetAndRefreshBodyData(_bodyDatas);

                //清楚该帧
                frame.Dispose();
                frame = null;
            }
        }
    }

    /// 
    /// 根据身体信息更新场景上的对象信息   创建  骨骼位置更新等
    /// 
    private void UpdateBodyObject()
    {
        //判断是否得到了身体信息
        if (_bodyDatas == null)
            return;

        _nowTrackedIDs.Clear();
        //记录当前追踪到的对象
        foreach (Body body in _bodyDatas)
        {
            if (body == null)
                continue;
            if (body.IsTracked)
                _nowTrackedIDs.Add(body.TrackingId);//trackingid是追踪到对象的唯一ID
        }

        //记录当前已经有的身体对象
        _nowCreatedBodyID = new List(_bodies.Keys);
        //遍历场景中已经创建的体感游戏对象  看看 是否当前已经不再被追踪了 动态的删除它
        foreach (ulong trackingID in _nowCreatedBodyID)
        {
            //当前kinect已经没有再扫描到对应对象了 那么在场景上删除它 并清楚记录
            if (!_nowTrackedIDs.Contains(trackingID))
            {
                Destroy(_bodies[trackingID].gameObject);
                _bodies.Remove(trackingID);
            }
        }

        //现在开始更新游戏对象信息 或者动态创建游戏对象
        foreach (Body body in _bodyDatas)
        {
            if (body == null)
                continue;
            //身体信息是否被追踪
            if (body.IsTracked)
            {
                //场景上没有对应游戏对象 创建对象
                if (!_bodies.ContainsKey(body.TrackingId))
                    CreateBodyObject(body.TrackingId);
                //更新对象的位置信息
                _bodies[body.TrackingId].RefreshBodyInfo(body);
            }
        }
    }

    /// 
    /// 动态 在场景上创建对象
    /// 
    /// 
    private void CreateBodyObject(ulong id)
    {
        GameObject obj = Instantiate(Resources.Load("RoleObject/GUJIA")) as GameObject;
        obj.name = "Body:" + id;
        KinectBodyControlLogic info = obj.GetComponent();
        _bodies.Add(id, info);
    }
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using System.Collections.Generic;

/// 
/// 身体控制逻辑
/// 
public class KinectBodyControlLogic : MonoBehaviour
{
    /// 
    /// 根骨骼
    /// 
    public GameObject SpineBase;
    /// 
    /// 腰部
    /// 
    public GameObject SpineMid;
    /// 
    /// 脖子
    /// 
    public GameObject Neck;
    /// 
    /// 头
    /// 
    public GameObject Head;
    /// 
    /// 左侧肩膀
    /// 
    public GameObject ShoulderLeft;
    /// 
    /// 左侧手肘
    /// 
    public GameObject ElbowLeft;
    /// 
    /// 左侧手腕
    /// 
    public GameObject WristLeft;
    /// 
    /// 左侧手掌
    /// 
    public GameObject HandLeft;
    /// 
    /// 右侧肩膀
    /// 
    public GameObject ShoulderRight;
    /// 
    /// 右侧手肘
    /// 
    public GameObject ElbowRight;
    /// 
    /// 右侧手腕
    /// 
    public GameObject WristRight;
    /// 
    /// 右侧手掌
    /// 
    public GameObject HandRight;
    /// 
    /// 左侧胯部
    /// 
    public GameObject HipLeft;
    /// 
    /// 左侧膝盖
    /// 
    public GameObject KneeLeft;
    /// 
    /// 左侧脚腕
    /// 
    public GameObject AnkleLeft;
    /// 
    /// 左侧脚掌
    /// 
    public GameObject FootLeft;
    /// 
    /// 右侧胯部
    /// 
    public GameObject HipRight;
    /// 
    /// 右侧膝盖
    /// 
    public GameObject KneeRight;
    /// 
    /// 右侧脚踝
    /// 
    public GameObject AnkleRight;
    /// 
    /// 右侧脚掌
    /// 
    public GameObject FootRight;
    /// 
    /// 肩部中央位置
    /// 
    public GameObject SpineShoulder;
    /// 
    /// 左侧指尖
    /// 
    public GameObject HandTipLeft;
    /// 
    /// 左侧拇指尖
    /// 
    public GameObject ThumbLeft;
    /// 
    /// 右侧指尖
    /// 
    public GameObject HandTipRight;
    /// 
    /// 右侧  侧拇指尖
    /// 
    public GameObject ThumbRight;


    /// 
    /// 是否是用Kinect镜像移动 为true 就像照镜子  为false就像背面控制模型
    /// 
    public bool IsMirror = true;

    //方向向量配对
    /// 
    /// 根骨骼
    /// 
    public JointType SpineBaseDir = JointType.SpineMid;
    /// 
    /// 腰部
    /// 
    public JointType SpineMidDir = JointType.SpineShoulder;
    /// 
    /// 脖子
    /// 
    public JointType NeckDir = JointType.Head;
    /// 
    /// 头
    /// 
    public JointType HeadDir = JointType.Head;
    /// 
    /// 左侧肩膀
    /// 
    public JointType ShoulderLeftDir = JointType.ElbowLeft;
    /// 
    /// 左侧手肘
    /// 
    public JointType ElbowLeftDir = JointType.WristLeft;
    /// 
    /// 左侧手腕
    /// 
    public JointType WristLeftDir = JointType.HandLeft;
    /// 
    /// 左侧手掌
    /// 
    public JointType HandLeftDir = JointType.HandTipLeft;
    /// 
    /// 右侧肩膀
    /// 
    public JointType ShoulderRightDir = JointType.ElbowRight;
    /// 
    /// 右侧手肘
    /// 
    public JointType ElbowRightDir = JointType.WristRight;
    /// 
    /// 右侧手腕
    /// 
    public JointType WristRightDir = JointType.HandRight;
    /// 
    /// 右侧手掌
    /// 
    public JointType HandRightDir = JointType.HandTipRight;
    /// 
    /// 左侧胯部
    /// 
    public JointType HipLeftDir = JointType.KneeLeft;
    /// 
    /// 左侧膝盖
    /// 
    public JointType KneeLeftDir = JointType.AnkleLeft;
    /// 
    /// 左侧脚腕
    /// 
    public JointType AnkleLeftDir = JointType.FootLeft;
    /// 
    /// 左侧脚掌
    /// 
    public JointType FootLeftDir = JointType.FootLeft;
    /// 
    /// 右侧胯部
    /// 
    public JointType HipRightDir = JointType.KneeRight;
    /// 
    /// 右侧膝盖
    /// 
    public JointType KneeRightDir = JointType.AnkleRight;
    /// 
    /// 右侧脚踝
    /// 
    public JointType AnkleRightDir = JointType.FootRight;
    /// 
    /// 右侧脚掌
    /// 
    public JointType FootRightDir = JointType.FootRight;
    /// 
    /// 肩部中央位置
    /// 
    public JointType SpineShoulderDir = JointType.Neck;
    /// 
    /// 左侧指尖
    /// 
    public JointType HandTipLeftDir = JointType.HandTipLeft;
    /// 
    /// 左侧拇指尖
    /// 
    public JointType ThumbLeftDir = JointType.ThumbLeft;
    /// 
    /// 右侧指尖
    /// 
    public JointType HandTipRightDir = JointType.HandTipRight;
    /// 
    /// 右侧  侧拇指尖
    /// 
    public JointType ThumbRightDir = JointType.ThumbRight;

    /// 
    /// 骨骼信息字典 对应Kinect的骨骼类型
    /// 
    private Dictionary _boneObjects;
    /// 
    /// 骨骼方向向量计算表示 用于骨骼方向向量计算
    /// 
    private Dictionary _boneDirTypes;

    //模型骨骼默认方向向量
    private Dictionary _bonesDir = new Dictionary();
    //模型骨骼默认旋转四元数信息
    private Dictionary _bonesRotation = new Dictionary();

    //根骨骼的方向向量  之后用于判断 人物朝向
    private Vector3 _rootDir;
    //上半身腰部骨骼的方向向量  用于之后判断 上半身朝向
    private Vector3 _upperBodyDir;

    //是否已经初始化
    private bool _isInit = false;

    void Awake()
    {
        //初始化骨骼信息
        InitBoneInfo();
        //初始化方向
        InitVector();
    }

    /// 
    /// 初始化骨骼字典信息
    /// 
    private void InitBoneInfo()
    {
        //初始化骨骼对象字典
        if(IsMirror)
        {
            _boneObjects = new Dictionary()
            {
                { JointType.SpineBase, SpineBase},
                { JointType.SpineMid, SpineMid},
                { JointType.Neck, Neck},
                { JointType.Head, Head},
                { JointType.ShoulderLeft, ShoulderRight},
                { JointType.ElbowLeft, ElbowRight},
                { JointType.WristLeft, WristRight},
                { JointType.HandLeft, HandRight},
                { JointType.ShoulderRight, ShoulderLeft},
                { JointType.ElbowRight, ElbowLeft},
                { JointType.WristRight, WristLeft},
                { JointType.HandRight, HandLeft},
                { JointType.HipLeft, HipRight},
                { JointType.KneeLeft, KneeRight},
                { JointType.AnkleLeft, AnkleRight},
                { JointType.FootLeft, FootRight},
                { JointType.HipRight, HipLeft},
                { JointType.KneeRight, KneeLeft},
                { JointType.AnkleRight, AnkleLeft},
                { JointType.FootRight, FootLeft},
                { JointType.SpineShoulder, SpineShoulder},
                { JointType.HandTipLeft, HandTipRight},
                { JointType.ThumbLeft, ThumbRight},
                { JointType.HandTipRight, HandTipLeft},
                { JointType.ThumbRight, ThumbLeft},
            };
        }
        else
        {
            _boneObjects = new Dictionary()
            {
                { JointType.SpineBase, SpineBase},
                { JointType.SpineMid, SpineMid},
                { JointType.Neck, Neck},
                { JointType.Head, Head},
                { JointType.ShoulderLeft, ShoulderLeft},
                { JointType.ElbowLeft, ElbowLeft},
                { JointType.WristLeft, WristLeft},
                { JointType.HandLeft, HandLeft},
                { JointType.ShoulderRight, ShoulderRight},
                { JointType.ElbowRight, ElbowRight},
                { JointType.WristRight, WristRight},
                { JointType.HandRight, HandRight},
                { JointType.HipLeft, HipLeft},
                { JointType.KneeLeft, KneeLeft},
                { JointType.AnkleLeft, AnkleLeft},
                { JointType.FootLeft, FootLeft},
                { JointType.HipRight, HipRight},
                { JointType.KneeRight, KneeRight},
                { JointType.AnkleRight, AnkleRight},
                { JointType.FootRight, FootRight},
                { JointType.SpineShoulder, SpineShoulder},
                 { JointType.HandTipLeft, HandTipLeft},
                { JointType.ThumbLeft, ThumbLeft},
                { JointType.HandTipRight, HandTipRight},
                { JointType.ThumbRight, ThumbRight},
            };
        }

        //初始化 各骨骼的默认方向向量
        _boneDirTypes = new Dictionary()
        {
             { JointType.SpineBase, SpineBaseDir},
             { JointType.SpineMid, SpineMidDir},
             { JointType.Neck, NeckDir},
             { JointType.Head, HeadDir},
             { JointType.ShoulderLeft, ShoulderLeftDir},
             { JointType.ElbowLeft, ElbowLeftDir},
             { JointType.WristLeft, WristLeftDir},
             { JointType.HandLeft, HandLeftDir},
             { JointType.ShoulderRight, ShoulderRightDir},
             { JointType.ElbowRight, ElbowRightDir},
             { JointType.WristRight, WristRightDir},
             { JointType.HandRight, HandRightDir},
             { JointType.HipLeft, HipLeftDir},
             { JointType.KneeLeft, KneeLeftDir},
             { JointType.AnkleLeft, AnkleLeftDir},
             { JointType.FootLeft, FootLeftDir},
             { JointType.HipRight, HipRightDir},
             { JointType.KneeRight, KneeRightDir},
             { JointType.AnkleRight, AnkleRightDir},
             { JointType.FootRight, FootRightDir},
             { JointType.SpineShoulder, SpineShoulderDir},
             { JointType.HandTipLeft, HandTipLeftDir},
             { JointType.ThumbLeft, ThumbLeftDir},
             { JointType.HandTipRight, HandTipRightDir},
             { JointType.ThumbRight, ThumbRightDir},
        };
    }

    /// 
    /// 初始化方向向量等信息
    /// 
    private void InitVector()
    {
        //这六个骨骼用于控制模型朝向  和 上半身朝向
        if( _boneObjects[JointType.HipRight] == null ||
            _boneObjects[JointType.HipLeft] == null ||
            _boneObjects[JointType.SpineBase] == null ||
            _boneObjects[JointType.ShoulderRight] == null ||
            _boneObjects[JointType.ShoulderLeft] == null ||
            _boneObjects[JointType.SpineMid] == null)
        {
            Debug.LogError("关键骨骼为空");
        }

        //根骨骼的方向向量 为 右侧胯骨减去左侧胯骨
        _rootDir = _boneObjects[JointType.HipRight].transform.position - _boneObjects[JointType.HipLeft].transform.position;
        //转换为相对于根节点的相对坐标
        _rootDir = _boneObjects[JointType.SpineBase].transform.InverseTransformDirection(_rootDir);

        //上半身骨骼的方向向量 为 右侧肩部减去左侧肩部
        _upperBodyDir = _boneObjects[JointType.ShoulderRight].transform.position - _boneObjects[JointType.ShoulderLeft].transform.position;
        //转换为相对于腰部点的相对坐标
        _upperBodyDir = _boneObjects[JointType.SpineMid].transform.InverseTransformDirection(_upperBodyDir);

        GameObject boneObj = null;
        foreach( JointType type in _boneObjects.Keys )
        {
            boneObj = _boneObjects[type];
            if (boneObj == null)
                continue;
            //初始旋转信息
            _bonesRotation.Add( type, boneObj.transform.localRotation );
            //初始方向向量信息
            _bonesDir.Add(type, _boneObjects[_boneDirTypes[type]].transform.position - _boneObjects[type].transform.position);
            //世界坐标转换为局部坐标
            _bonesDir[type] = boneObj.transform.InverseTransformDirection(_bonesDir[type]);
        }
        _isInit = true;
    }

    /// 
    /// 跟新身体信息
    /// 
    /// 
    public void RefreshBodyInfo(Body body)
    {
        if (!_isInit)
            return;
        //Kinect得到的Body骨骼信息
        Windows.Kinect.Joint sourceJoint;
        GameObject boneObj;
        Vector3 boneDir;
        Vector3 targetDir;
        //更新模型的所有骨骼信息
        foreach ( JointType jt in _boneObjects.Keys )
        {
            boneObj = _boneObjects[jt];
            if (boneObj == null)
                continue;
            //得到Kinect的body信息中对应骨骼点信息
            sourceJoint = body.Joints[jt];
            //初始化角度
            boneObj.transform.localRotation = _bonesRotation[jt];

            //初始的方向向量
            boneDir = _bonesDir[jt];
            //当前kinect中对应骨骼的方向向量
            targetDir = GetVector3FromJoint(body.Joints[_boneDirTypes[jt]]) - GetVector3FromJoint(sourceJoint);
            //Kinect坐标转U3D世界坐标
            targetDir = transform.TransformDirection(targetDir);
            //从U3D世界坐标转相对坐标
            targetDir = boneObj.transform.InverseTransformDirection(targetDir);
            //得到相对角度变换
            Quaternion nowQuat = Quaternion.FromToRotation(boneDir, targetDir);

            //根骨骼的方向变化
            if(jt == JointType.SpineBase)
            {
                //之前得到的胯骨平行方向向量
                boneDir = _rootDir;
                //得到当前kinect的肩部骨骼方向向量位置
                targetDir = GetVector3FromJoint(body.Joints[JointType.HipRight]) - GetVector3FromJoint(body.Joints[JointType.HipLeft]);
                //坐标转换 因为不希望在x轴上旋转 让角色平稳的站在地上 所以 让其在x轴上取投影
                targetDir = Vector3.Project(targetDir, transform.right);
                targetDir = transform.TransformDirection(targetDir);
                targetDir = boneObj.transform.InverseTransformDirection(targetDir);

                nowQuat *= Quaternion.FromToRotation(boneDir, targetDir);
            }
            //上半身骨骼的方向变化
            else if (jt == JointType.SpineMid)
            {
                //上半身平行位置
                boneDir = _upperBodyDir;
                //得到当前kinect的肩部骨骼方向向量位置
                targetDir = GetVector3FromJoint(body.Joints[JointType.ShoulderRight]) - GetVector3FromJoint(body.Joints[JointType.ShoulderLeft]);
                //坐标转换
                targetDir = transform.TransformDirection(targetDir);
                targetDir = boneObj.transform.InverseTransformDirection(targetDir);

                nowQuat *= Quaternion.FromToRotation(boneDir, targetDir);
            }

            //开始转
            nowQuat = Quaternion.Lerp(Quaternion.identity, nowQuat, 1);
            boneObj.transform.localRotation = boneObj.transform.localRotation * nowQuat;

            //更新人物位置
            if (jt == JointType.SpineBase)
                this.transform.position = GetVector3FromJoint(sourceJoint);
        }
    }

    /// 
    /// 把Kinect中的骨骼关节点 转换为U3D中的Vector3
    /// 
    /// 
    /// 
    private Vector3 GetVector3FromJoint(Windows.Kinect.Joint joint)
    {
        return new Vector3(joint.Position.X, joint.Position.Y, IsMirror ? joint.Position.Z : -joint.Position.Z);
    }
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using UnityEngine.UI;

/// 
/// 摄像机信息 主要原理就是通过Kinect得到图像RGB数据 然后在Texture上绘制
/// 基本逻辑和获取身体信息相同
/// 
public class CamerColorView : MonoBehaviour
{
    //Kinect对象
    private KinectSensor _kinectSensor;
    //颜色信息读取流
    private ColorFrameReader _colorReader;
    private Texture2D _textureInfo;
    private byte[] _colorData;

    private RawImage _cameraMaterial;

	// Use this for initialization
	void Start ()
    {
        _cameraMaterial = this.gameObject.GetComponent();
        _cameraMaterial.uvRect = new Rect(0, 0, 1, -1);
        _kinectSensor = KinectSensor.GetDefault();
        if (_kinectSensor == null)
            return;

        _colorReader = _kinectSensor.ColorFrameSource.OpenReader();

        //初始化图片信息
        FrameDescription color = _kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
        _textureInfo = new Texture2D(color.Width, color.Height, TextureFormat.RGBA32, false);
        _colorData = new byte[color.BytesPerPixel * color.LengthInPixels];

        if (!_kinectSensor.IsOpen)
            _kinectSensor.Open();
	}
	
	// Update is called once per frame
	void Update ()
    {
        if (_colorReader == null)
            return;

        ColorFrame frame = _colorReader.AcquireLatestFrame();
        if (frame == null)
            return;
        //存储rgb信息
        frame.CopyConvertedFrameDataToArray(_colorData, ColorImageFormat.Rgba);
        //texture加载信息
        _textureInfo.LoadRawTextureData(_colorData);
        //应用
        _textureInfo.Apply();

        frame.Dispose();
        frame = null;
      
        //设置显示信息
        _cameraMaterial.texture = _textureInfo;
    }
}