Kinect2-手势控制鼠标

废话不多说，直奔主题！

1.环境

  1.系统 Windows 8.x
  2.Kinect for Windows SDK v2.0（要求系统 Windows 8.x）点这里
  3.设备 xbox二代

2.手势控制鼠标

KinectApp.h

#pragma once
#include "stdafx.h"
class KinectApp
{
    //分辨率
    int width ;
    int height;
    //手臂移动的距离
    int X;
    int Y;
    int Z;
    //可触发指针移动的最小距离
    int LR;
    int TB;
    //滑动手势距离 手势
    int sliding;
    int sleeptime;
    //触发手势滑动的时间 距离
    int sTime;
    int sLong;
    // 滑动时间
    int panTime;
    // 点击间隔时间
    int clickTime;
    // 未追踪Id
    UINT64 UnknownTrackingId;
private:
    /*是否初始化成功*/
    BOOL isInitSdk;
    /**传感器是否继续运行 */
    BOOL isRun;

    BodyRect bodyRect;
    BodyRect panstartRect;
    BodyRect panendRect;
private:
    // Kinect v2 传感器
    IKinectSensor*    m_pKinect ;
    IBodyFrameReader* bodyReader;
private:
    /** 回掉的函数指针*/
    ActionCallBack _actionCallBack;
    ErrorCallBack  _errorCallBack;
    PanActionCallBack _panActionCallBack;

    void m_actionCallBack(int x,int y,int z,ActionType type);
    void m_errorCallBack(string codeStr);
    void m_panActionCallBack();
    void m_click(int x,int y,int z,ActionType type);
private:
    /** 初始化SDK*/
    inline HRESULT initSDK();
    /** 传感器是否可用*/
    inline BOOL isAvailable();
    /** 初始化身体数据源*/
    inline HRESULT initBodySource();
    /**获取身体数据 */
    inline void getBody();
    /**获取身体数据 */
    inline IBody* getTrackedBody(IBody* ppBodies[]);
    /**获取追踪对象的手势 */
    inline void getTrackedBodyJoints(IBody* pBody);
    /** */
    inline BodyRect getHandRect(Joint joints[],JointType JointTypeArray[]);
    /** */
    inline void resetRect();
    /**人物是不是在范围内 */
    inline bool isInScope(IBody* pBody);

public:
    KinectApp(void);
    ~KinectApp(void);
    void open();
    void close();
    void openBodyReader();
    void sTimeTimer();
public:
    /**设置回掉函数*/
    void m_setActionCallBack(ActionCallBack call);
    void m_setErrorActionCallBack(ErrorCallBack call);
    void m_setPanActionCallBack(PanActionCallBack call);
};

KinectApp.cpp

#include "KinectApp.h"


KinectApp::KinectApp(void){

    width = 1920;
    height = 1200;
    X = 240;
    Y = 200;
    Z = 2800;
    LR = 12;
    TB = 8;
    sliding = 500;
    sleeptime = 0;
    sTime = 500;
    sLong = 100;

    panTime = 0;
    clickTime = 0;
    UnknownTrackingId = 7;
    resetRect();

    isInitSdk = false;
    isRun = false;

    m_pKinect = nullptr;
    bodyReader = nullptr;

    if (SUCCEEDED(initSDK()))
    {
        isInitSdk = true;
    }

}

KinectApp::~KinectApp(void){
    // 优雅地关闭Kinect
    if (m_pKinect){
        m_pKinect->Close();
    }
    // 释放语音识别对象
    SafeRelease(m_pKinect);
    SafeRelease(bodyReader);
}

/** 传感器是否可用*/
inline BOOL KinectApp::isAvailable() {
    if (isInitSdk == false)
    {
        m_errorCallBack("initsdk failed");
        return false;
    }
    BOOLEAN _isAvailable = false;
    m_pKinect->get_IsAvailable(&_isAvailable);
    return _isAvailable;
}

inline HRESULT KinectApp::initSDK()
{
    HRESULT hr = S_OK;
    if (m_pKinect == nullptr)
    {
        hr = GetDefaultKinectSensor(&m_pKinect);
    }
    if (!SUCCEEDED(hr))
    {
        m_errorCallBack("GetDefaultKinectSensor failed");
        return hr;
    }
    hr = m_pKinect->Open();
    if (!SUCCEEDED(hr))
    {
        m_errorCallBack("Kinect Open failed");
        return hr;
    }
    hr = initBodySource();
    if (!SUCCEEDED(hr))
    {
        m_errorCallBack("initBodySource failed");
        return hr;
    }
    return hr;
}

inline HRESULT KinectApp::initBodySource()
{
    //添加身体数据源
    IBodyFrameSource* bodySource = nullptr;
    HRESULT hr = m_pKinect->get_BodyFrameSource(&bodySource);
    if (!SUCCEEDED(hr))
    {
        m_errorCallBack("get_BodyFrameSource failed");
        return hr;
    }
    //读取身体数据源
    INT32 nBodyNum = 0;
    bodySource->get_BodyCount(&nBodyNum);
    hr = bodySource->OpenReader(&bodyReader); // 准备读取body数据
    if (!SUCCEEDED(hr))
    {
        m_errorCallBack("OpenReader failed");
        return hr;
    }
    return hr;
}

/**获取身体数据 */
inline void KinectApp::getBody(void){

    Sleep(sleeptime);
    // 获取最近的一帧数据
    IBodyFrame* bodyf = nullptr;
    bodyReader->AcquireLatestFrame(&bodyf);
    if (!bodyf)
    {
        return;
    }
    // 更新所有人身体数据
    IBody* ppBodies[BODY_COUNT] = { 0 };
    bodyf->GetAndRefreshBodyData(BODY_COUNT, ppBodies);

    IBody*pbody = getTrackedBody(ppBodies);
    if (pbody != nullptr)
    {
        getTrackedBodyJoints(pbody);
    }

    SafeRelease(bodyf);
}


inline IBody* KinectApp::getTrackedBody(IBody* ppBodies[])
{
    UINT64 temptrckingId = UnknownTrackingId;
    //遍历每个人的信息
    IBody *ppBody = nullptr;
    for (int i = 0; i < BODY_COUNT; ++i)
    {
        IBody *pBody = ppBodies[i]; // 轮询每个人的信息
        if (pBody)
        {
            // 检测是否被跟踪，即是否有这个人
            BOOLEAN bTracked = false;
            HRESULT hr = pBody->get_IsTracked(&bTracked);
            if (bTracked && SUCCEEDED(hr))
            {
                bool inscope = isInScope(pBody);
                if (inscope)
                {
                    UINT64 tid = UnknownTrackingId;
                    pBody->get_TrackingId(&tid);
                    if (ppBody == nullptr)
                    {
                        ppBody = ppBodies[i];
                        temptrckingId = tid;
                    }
                    if (bodyRect.trackingId == tid)
                    {
                        return pBody;
                    }
                }
            }
        }
    }

    panstartRect.trackingId = temptrckingId;
    panendRect.trackingId = temptrckingId;
    bodyRect.trackingId = temptrckingId;

    return ppBody;
}

inline void KinectApp::getTrackedBodyJoints(IBody* pBody)
{
    Joint joints[JointType_Count];
    HRESULT hr = pBody->GetJoints(_countof(joints), joints);
    if (SUCCEEDED(hr))
    {
        // 获取右手状态
        HandState rightHandState = HandState_Unknown;
        pBody->get_HandRightState(&rightHandState);
        ActionType type = (ActionType)rightHandState;
        if (type == ActionType_NotTracked)
        {
            return;
        }
        // 获取骨骼坐标数组
        JointType JointTypeArray[] = {JointType_HandRight,JointType_WristRight,JointType_HandTipRight};

        BodyRect aRect = getHandRect(joints,JointTypeArray);

        int x = aRect.X;
        int y = aRect.Y;
        int z = aRect.Z;
        if (z>Z)
        {
            resetRect();
            return;
        }
        if (abs(x-bodyRect.X) > LR || abs(y-bodyRect.Y) > TB)
        {
            bodyRect.X = x;
            bodyRect.Y = y;
            bodyRect.Z = z;
            SetCursorPos(x,y);
            m_actionCallBack(x,y,z,type);
        }

        if(bodyRect.type != type){
            m_click(x,y,z,type);
        }
    }
}

inline bool KinectApp::isInScope(IBody* pBody)
{
    Joint joints[JointType_Count];
    HRESULT hr = pBody->GetJoints(_countof(joints), joints);
    if (SUCCEEDED(hr))
    {
        JointType JointTypeArray[] = {JointType_Head};
        Joint headJoint = joints[JointTypeArray[0]];
        printf("%f \n",headJoint.Position.Z);
        if (headJoint.Position.Z > 2.8)
        {
            return false;
        }
        return true;
    }
    return hr;
}

inline BodyRect KinectApp::getHandRect(Joint joints[],JointType JointTypeArray[])
{
    static vector bodyRectArray;
    float xx = 0;
    float yy = 0;
    float zz = 0;
    int len = sizeof(JointTypeArray) / sizeof(JointTypeArray[0]);
    for (int i = 0; i < len; i++)
    {
        Joint rightJoint = joints[JointTypeArray[i]];
        xx += rightJoint.Position.X;
        yy += rightJoint.Position.Y;
        zz += rightJoint.Position.Z;
    }
    xx = xx/len;
    yy = yy/len;
    zz = zz/len;
    //计算像素 距离
    int x = (int)((xx * 1000/X + 1) *(width/2));
    int y = (int)((1 - yy * 1000/Y) *(height/2));
    int z = (int)(zz * 1000);
    BodyRect aRect = {x,y,z,7,ActionType_Unknown};

    if (bodyRectArray.size() >= 6)
    {
        vector::iterator temp = bodyRectArray.begin();
        bodyRectArray.erase(temp);
    }
    bodyRectArray.push_back(aRect);

    int xxx = 0;
    int yyy = 0;
    int zzz = 0;
    for (int i = 0; i < bodyRectArray.size(); i++)
    {
        BodyRect tempRect = bodyRectArray[i];
        xxx += tempRect.X;
        yyy += tempRect.Y;
        zzz += tempRect.Z;
    }
    xxx = xxx/bodyRectArray.size();
    yyy = yyy/bodyRectArray.size();
    zzz = zzz/bodyRectArray.size();

    BodyRect rRect = {xxx,yyy,zzz,7,ActionType_Unknown};

    return rRect;
}

inline void KinectApp::resetRect()
{
    bodyRect.X = bodyRect.Y = bodyRect.Z  = 0;
    bodyRect.trackingId = UnknownTrackingId;
    bodyRect.type = ActionType_NotTracked;

    panstartRect.X = panstartRect.Y = panstartRect.Z  = 0;
    panstartRect.trackingId = UnknownTrackingId;
    panstartRect.type = ActionType_NotTracked;

    panendRect.X = panendRect.Y = panendRect.Z  = 0;
    panendRect.trackingId = UnknownTrackingId;
    panendRect.type = ActionType_NotTracked;

    panTime = 0;
}

void KinectApp::m_click(int x,int y,int z,ActionType type){
    if (type == ActionType_Closed)
    {
        if(clickTime >=1500)
        {
                mouse_event(MOUSEEVENTF_LEFTDOWN,0,0,0,0);
                mouse_event(MOUSEEVENTF_LEFTUP,0,0,0,0);//松开左键
                clickTime = 0;
        }
    }
    bodyRect.type = type;
}

void KinectApp::open()
{
    isRun = true;
    thread th(&KinectApp::openBodyReader,this);
    th.detach();
    thread th2(&KinectApp::sTimeTimer,this);
    th2.detach();
}

void KinectApp::close()
{
    isRun = false;
}

void KinectApp::openBodyReader()
{
    printf("start Tracking person \n");
    while (true)
    {
        if (isRun == false)
        {
            break;
        }
        if (isAvailable())
        {
            this->getBody();
        }else
        {
            Sleep(1000);
            m_errorCallBack("waitting kinect connect");
        }
    }
}

void KinectApp::sTimeTimer()
{
    while (true)
    {
        Sleep(100);
        clickTime+=100;
    }
}

void KinectApp::m_setActionCallBack(ActionCallBack call)
{
    _actionCallBack = call;
}

void KinectApp::m_setErrorActionCallBack(ErrorCallBack call)
{
    _errorCallBack = call;
}

void KinectApp::m_setPanActionCallBack(PanActionCallBack call)
{
    _panActionCallBack = call;
}

void KinectApp::m_actionCallBack(int x,int y,int z,ActionType type)
{
    if (_actionCallBack)
    {
        BodyRect re = {x,y,z,bodyRect.trackingId,type};
        _actionCallBack(re);
    }
}

void KinectApp::m_errorCallBack(string codeStr)
{
    if (_errorCallBack)
    {
        _errorCallBack(codeStr);
    }
}

void KinectApp::m_panActionCallBack()
{
    printf("滑动 \n");
    int x = panstartRect.X - panendRect.X;
    int y = panstartRect.Y - panendRect.Y;
    printf("%d,%d",x,y);
    if (abs(y) >= sLong)
    {
        PanActionType statusType = PanActionType_Up;
        if (y < 0)
        {
            statusType = PanActionType_Bottom;
        }
        POINT point;
        BOOL hr = GetCursorPos(&point);
        if(hr){
            HWND hWnd = WindowFromPoint(point);
            if (statusType == PanActionType_Up)
            {
                SendMessage(hWnd,WM_MOUSEWHEEL,-WHEEL_DELTA<<16,0);
            }else if (statusType == PanActionType_Bottom)
            {
                SendMessage(hWnd,WM_MOUSEWHEEL,WHEEL_DELTA<<16,0);
            }
        }

        if (_panActionCallBack)
        {
            _panActionCallBack((PanActionType)statusType);
        }
    }
}

stdafx.h

#pragma once
#pragma warning(disable : 4996)

#define WIN32_LEAN_AND_MEAN     // Exclude rarely-used stuff from Windows headers
// Windows Header Files:
#include 
#include 
// Kinect
#include 
//
#include 
#include 
// C++ 
#include 

#include 
#include 
#include 

#include
#include
#include
#include 

using namespace std;

template
inline void SafeRelease(Interface *&pInterfaceToRelease)
{
    if (pInterfaceToRelease != NULL)
    {
        pInterfaceToRelease->Release();
        pInterfaceToRelease = NULL;
    }
}

#ifndef Assert
#if defined( DEBUG ) || defined( _DEBUG )
#define Assert(b) if (!(b)) {OutputDebugStringA("Assert: " #b "\n");}
#else
#define Assert(b)
#endif //DEBUG || _DEBUG
#endif

//#if defined( DEBUG ) || defined( _DEBUG )
//#define MYTRACE(a) _cwprintf(a)
//#else
//#define MYTRACE(a)
//#endif

#pragma comment ( lib, "d2d1.lib" )
#pragma comment ( lib, "windowscodecs.lib" )
#pragma comment ( lib, "dwmapi.lib" )
#pragma comment ( lib, "Msdmo.lib" )
#pragma comment ( lib, "dmoguids.lib" )
#pragma comment ( lib, "amstrmid.lib" )
#pragma comment ( lib, "avrt.lib" )
#pragma comment ( lib, "dwrite.lib" )
#pragma comment ( lib, "kinect20.lib" )

#pragma comment (lib,  "sapi.lib")
#pragma comment (lib,  "ole32.lib")//需要调用ole32.dll

enum ActionType
{
    ActionType_Unknown = 0,
    ActionType_NotTracked = 1,
    ActionType_Open = 2,
    ActionType_Closed = 3,
    ActionType_Lasso = 4
};

enum PanActionType
{
    PanActionType_Up = 0,
    PanActionType_Bottom = 1,
    PanActionType_Left = 2,
    PanActionType_Right = 3
};

enum AudioCallStatus
{
    AudioCallStatus_Success = 0,
    AudioCallStatus_Faild = 1
};

enum KinectActionType
{
    KinectActionType_Body = 0,
    KinectActionType_Faild = 1,
};

typedef struct _BodyRect
{
    int X;
    int Y;
    int Z;
    UINT64 trackingId;
    ActionType type;
} BodyRect;

typedef void (__stdcall *ErrorCallBack)(string codeStr);
typedef void (__stdcall *ActionCallBack)(BodyRect bRect);
typedef void (__stdcall *PanActionCallBack)(PanActionType type);
typedef void (__stdcall *AudioCallBack)(string str,AudioCallStatus status);


#define lengthof(a) (sizeof(a)/sizeof(*a))

3.测试代码

#include "KinectApp.h"
#include 
#include "MKSpeech.h"

void __stdcall onActionCallBack(BodyRect bRect)
{
    printf("Person %d : X：%d Y：%d Z:%d State：%d \n", 0, bRect.X,bRect.Y,bRect.Z,bRect.type); 
}

void __stdcall onErrorCallBack(string codeStr)
{
    std::cout << codeStr << std::endl;
}

void __stdcall onPanActionCallBack(PanActionType type)
{
    printf("%c",type); 
}


void __stdcall onAudioCallBack(string str,AudioCallStatus status)
{

    printf("%d ",status); 
    
    std::cout << str << std::endl;
}

KinectApp *app;
MKSpeech *speech;

int main()
{
    
    if (SUCCEEDED(CoInitialize(NULL)))
    {
        /*app = new KinectApp();
        app->m_setActionCallBack(onActionCallBack);
        app->m_setErrorActionCallBack(onErrorCallBack);
        app->m_setPanActionCallBack(onPanActionCallBack);
        app->open();*/

        speech = new MKSpeech();
        speech->m_setErrorActionCallBack(onErrorCallBack);
        speech->m_setAudioCallBack(onAudioCallBack);
        while (true)
        {
            int a = rand()%2;
            printf("%d",a);
            speech->open(a);
            system("pause");
        }
    }
    
}

4.几项注意

    1.手势获取人物每次都是6人，获取图像速度为30帧，手势存在误差，约在7mm内。
    2.语音识别为指令性语音识别，识别指令相似度越高，识别准确率越低。
    3.本代码开发工具为Visual Studio。
    4.如有问题、建议可联系[email protected]。
    5.手势识别为防止鼠标抖动每次都取6个历史位置的平均值。
    6.如代码不可运行请看下一篇章语音识别。

本文参考文献：

    [https://blog.csdn.net/dustpg/column/info/k4w2dn](https://blog.csdn.net/dustpg/column/info/k4w2dn)

代码地址：链接: https://pan.baidu.com/s/1aY8S2VWOBIsW-JbAqcdEow 提取码: kujw