kinect结合ngui开发

自从微软发布了kinect2.0以后，自带的kinect with unity sdk开发包用起来要爽很多，给我们使用unity开发体感游戏带来了方便，其中是对于手掌的识别，使得我们开发游戏界面时处理起点击和按住拖动等事件起来得心应手。

下面我们来看下如何用kinect结合ngui来开发界面：

最开始我想到的是使用一个UITexture绑定Raycast来模拟鼠标，后来发现虽然用来处理点击事件很好用，但是对于ngui对象的拖动来说要处理的事情太多了，代码如下，可以用来参考：

RaycastHit[] hits = Physics.RaycastAll(transform.position, transform.forward);
if (hits.Length >= 1)
{
	int maxDepth = 0;
	int i = 0;
	for (int b = 0; b < hits.Length; b++)
	{
		GameObject go = hits[b].collider.gameObject;
		UIWidget w = go.GetComponent();
		if (w != null && w.depth > maxDepth)
		{
			maxDepth = w.depth;
			i = b;
		}
	}
	if (preHandState == Windows.Kinect.HandState.Open && handState == Windows.Kinect.HandState.Closed)
	{
		UICamera.Notify(hits[i].collider.gameObject, "OnPress", true);
		clickStartObject = hits[i].collider.gameObject;
	}
	if (preHandState == Windows.Kinect.HandState.Closed && handState == Windows.Kinect.HandState.Open)
	{
		UICamera.Notify(hits[i].collider.gameObject, "OnPress", false);
		if (hits[i].collider.gameObject == clickStartObject)
		{
			UICamera.Notify(hits[i].collider.gameObject, "OnClick", null);
			Debug.Log("Click:" + hits[i].collider.name);
		}
	}

	if (hits[i].collider.gameObject.GetComponent() != null)
	{
		if (preHandState == Windows.Kinect.HandState.Open && handState == Windows.Kinect.HandState.Closed)
		{
			dragStart = true;
			dragStartPosition = new Vector2(_Hand.Position.X * 1920, _Hand.Position.Y * 1080);
			UICamera.MouseOrTouch kinectMouse = new UICamera.MouseOrTouch();
			kinectMouse.delta = dragStartPosition - kinectMouse.pos;
			kinectMouse.pos = dragStartPosition;
			kinectMouse.current = hits[i].collider.gameObject;
		}
		if (preHandState == Windows.Kinect.HandState.Closed && handState == Windows.Kinect.HandState.Closed && dragStart)
		{
			Vector2 delta = new Vector2(_Hand.Position.X * 1920, _Hand.Position.Y * 1080) - dragStartPosition;
			dragStartPosition += delta;
			UICamera.Notify(hits[i].collider.gameObject, "OnDrag", delta);
			Debug.Log("Drag:" + hits[i].collider.name);
		}
		if (preHandState == Windows.Kinect.HandState.Closed && handState == Windows.Kinect.HandState.Open && dragStart)
		{
			dragStart = false;
		}
	}
}
if (handState == Windows.Kinect.HandState.Closed || handState == Windows.Kinect.HandState.Open)
{
	preHandState = handState;
}

其中的OnDrag事件是没有任何响应的，后来通过查看ngui的UICamera源码发现，要使用OnDrag事件还需要设置UICamera中的众多参数，由于太过麻烦，我开始从UICamera 源码上动起脑经来。通过查看源码可以发现UICamera其实是自带了鼠标，触摸，控制器三种方式的事件处理，于是我在其中新增了Kinect的事件处理：

	public enum ControlScheme
	{
	    Mouse,
	    Touch,
	    Controller,
            Kinect,
	}

public bool useTouch = true;

在源码中我们还可以看到，所有的事件处理都是通过MouseOrTouch对象来实现的，于是我增加了：

static public MouseOrTouch kinectMouse = new MouseOrTouch();

通过对比发现kinectMouse和mouse事件处理的方式方式最为相似，于是我们使用mouse来做为模版进行修改来实现我们自己的kinectMouse，修改后的代码：

public void ProcessKinectMouse()
    {
        lastTouchPosition = KinectController.kinectMousePosition;
        kinectMouse.delta = lastTouchPosition - kinectMouse.pos;
        kinectMouse.pos = lastTouchPosition;
        bool posChanged = mMouse[0].delta.sqrMagnitude > 0.001f;

        bool isPressed = false;
        bool justPressed = false;

        if (KinectController.handClose)
        {
            currentScheme = ControlScheme.Kinect;
            justPressed = true;
            isPressed = true;
        }
        else if (KinectController.handClosed)
        {
            currentScheme = ControlScheme.Kinect;
            isPressed = true;
        }

        if (isPressed || posChanged || mNextRaycast < RealTime.time)
        {
            mNextRaycast = RealTime.time + 0.02f;
            if (!Raycast(KinectController.kinectMousePosition, out lastHit)) hoveredObject = fallThrough;
            if (hoveredObject == null) hoveredObject = genericEventHandler;
            kinectMouse.current = hoveredObject;
        }

        bool highlightChanged = (kinectMouse.last != kinectMouse.current);
        if (highlightChanged) currentScheme = ControlScheme.Kinect;

        if (isPressed)
        {
            // A button was pressed -- cancel the tooltip
            mTooltipTime = 0f;
        }
        else if (posChanged && (!stickyTooltip || highlightChanged))
        {
            if (mTooltipTime != 0f)
            {
                // Delay the tooltip
                mTooltipTime = RealTime.time + tooltipDelay;
            }
            else if (mTooltip != null)
            {
                // Hide the tooltip
                ShowTooltip(false);
            }
        }

        if ((justPressed || !isPressed) && mHover != null && highlightChanged)
        {
            currentScheme = ControlScheme.Kinect;
            if (mTooltip != null) ShowTooltip(false);
            Notify(mHover, "OnHover", false);
            mHover = null;
        }


        bool pressed = KinectController.handClose;
        bool unpressed = KinectController.handOpen;

        if (pressed || unpressed) currentScheme = ControlScheme.Kinect;

        currentTouch = kinectMouse;
        currentTouchID = -1;

        // We don't want to update the last camera while there is a touch happening
        if (pressed) currentTouch.pressedCam = currentCamera;
        else if (currentTouch.pressed != null) currentCamera = currentTouch.pressedCam;

        // Process the mouse events
        ProcessTouch(pressed, unpressed);
        
        currentTouch = null;

        // If nothing is pressed and there is an object under the touch, highlight it
        if (!isPressed && highlightChanged)
        {
            currentScheme = ControlScheme.Kinect;
            mTooltipTime = RealTime.time + tooltipDelay;
            mHover = kinectMouse.current;
            Notify(mHover, "OnHover", true);
        }

        // Update the last value
        kinectMouse.last = kinectMouse.current;
    }

接下来我们再Update()中添加一行代码就可以开始处理我们的kinectMouse事件了

        if (useKinect) ProcessKinectMouse();

关于Kinect的部分就不细说了，就是简单的使用手掌开合来模拟鼠标的按住放开，直接上代码：

using UnityEngine;
using System.Collections;


public class KinectController : MonoBehaviour
{
    public Texture[] handTextures;
    public enum HandType
    {
        Left,
        Right,
    }
    public HandType handType;
    public static Vector3 handPosition;
    public static Vector3 kinectMousePosition;
    public static bool handClose;
    public static bool handOpen;
    public static bool handClosed;


    BodySourceManager bodySource;
    Windows.Kinect.Joint hand;
    Windows.Kinect.HandState handState;
    Windows.Kinect.HandState lastHandState;


    GameObject clickStartObject;
    bool dragStart;
    Vector2 dragStartPosition;


    // Use this for initialization
    void Start()
    {
        gameObject.GetComponent().mainTexture = handTextures[0];
        bodySource = GameObject.Find("KinectManager").GetComponent();
        DontDestroyOnLoad(transform.parent.gameObject);
    }


    // Update is called once per frame
    void Update()
    {
        handClosed = false;
        handClose = false;
        handOpen = false;


        gameObject.GetComponent().mainTexture = handTextures[0];


        Windows.Kinect.Body[] data = bodySource.GetData();
        if (data == null)
        {
            return;
        }
        foreach (var body in data)
        {
            if (body == null)
            {
                return;
            }
            if (body.IsTracked)
            {
                if (handType == HandType.Left)
                {
                    hand = body.Joints[Windows.Kinect.JointType.HandLeft];
                    handState = body.HandLeftState;
                }
                else
                {
                    hand = body.Joints[Windows.Kinect.JointType.HandRight];
                    handState = body.HandRightState;
                }


                handPosition = new Vector3(hand.Position.X * 1920, hand.Position.Y * 1080, 0);
                transform.localPosition = handPosition;


                kinectMousePosition = UICamera.currentCamera.WorldToScreenPoint(transform.position);
                Vector3 mousepos = Input.mousePosition;


                if (lastHandState == Windows.Kinect.HandState.Open && handState == Windows.Kinect.HandState.Closed)
                {
                    handClose = true;
                    Debug.Log("handClose");
                }
                else if (lastHandState == Windows.Kinect.HandState.Closed && handState == Windows.Kinect.HandState.Open)
                {
                    handOpen = true;
                    Debug.Log("handOpen");
                }
                else if (lastHandState == Windows.Kinect.HandState.Closed && handState == Windows.Kinect.HandState.Closed)
                {
                    gameObject.GetComponent().mainTexture = handTextures[1];
                    handClosed = true;
                }


                if (handState == Windows.Kinect.HandState.Closed || handState == Windows.Kinect.HandState.Open)
                {
                    lastHandState = handState;
                }
            }
        }
    }
}