利用OPENNI的用户数据生成器和OPENCV判断手势的演示

利用OPENNI的用户数据生成器和OPENCV判断手势的演示

1，原理：读入KINECT深度数据和用户数据，根据用户数据中最接近点附近的图像找到轮廓，与轮廓模板比较，找到HU矩阵最小的为匹配结果

2，基础：OPENNI, OPENCV2.2，上一篇“利用KINECT+OPENCV检测手势的演示程序”以及http://blog.163.com/gz_ricky/blog/static/182049118201122311118325/
的例程基础上修改

3，结果：对手位置的判断有所改善，但处理速度明显变慢。仅仅用于演示利用OPENCV+OPENNI编程，对结果精度，处理速度等没有优化，没有进行错误处理，仅供参考

 

// HandDetectByUser.cpp : 定义控制台应用程序的入口点。 // #include "stdafx.h" #include <stdlib.h> #include <iostream> #include <string> #include <XnCppWrapper.h> #include <opencv2/opencv.hpp> using namespace std; using namespace cv; //#define SAMPLE_XML_PATH "../../Data/SamplesConfig.xml" //Distance area (mm) //int g_forwardDist = 50; int g_backwardDist = 100; //全局模板轮廓 vector<vector<Point>> g_TemplateContours; //模板个数 int g_handTNum = 6; void CheckOpenNIError( XnStatus eResult, string sStatus ) { if( eResult != XN_STATUS_OK ) { cerr << sStatus << " Error: " << xnGetStatusString( eResult ) << endl; return; } } // Callback: A new user find void XN_CALLBACK_TYPE User_NewUser(xn::UserGenerator& generator, XnUserID nId, void* pCookie) { printf("New User %d/n", nId); } // Callback: An existing user was lost void XN_CALLBACK_TYPE User_LostUser(xn::UserGenerator& generator, XnUserID nId, void* pCookie) { printf("Lost user %d/n", nId); } //载入模板的轮廓 void init_hand_template() { //int handTNum = 10; string temp = "HandTemplate/"; int i = 0; for(i=0; i<g_handTNum; i++) { stringstream ss; ss << i << ".bmp"; string fileName = temp + ss.str(); //读入灰度图像 Mat src = imread(fileName, 0); if(!src.data) { printf("未找到文件: %s/n", fileName); continue; } vector<vector<Point>> contours; vector<Vec4i> hierarchy; findContours(src, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); //findContours(src, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE); g_TemplateContours.push_back(contours[0]); } } //模板匹配手 int hand_template_match(Mat& hand) { //int handTNum = 10; int minId = -1; double minHu = 1; double hu; int method = CV_CONTOURS_MATCH_I1; //match_num = 0; for(int i=0; i<g_handTNum; i++){ Mat temp(g_TemplateContours.at(i)); hu = matchShapes(temp, hand, method, 0); //找到hu矩最小的模板 if(hu < minHu){ minHu = hu; minId = i; } //printf("%f ", hu); } //显示匹配结果 int Hmatch_value = 25;//模板匹配系数 if(minHu<((double)Hmatch_value)/100) return minId; else return -1; } void findHand(Mat& src, Mat& dst) { vector<vector<Point>> contours; vector<Vec4i> hierarchy; //找到外部轮廓 //findContours(src, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); findContours(src, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); //CV_CHAIN_APPROX_NONE); //findContours(src, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); Mat dst_r = Mat::zeros(src.rows, src.cols, CV_8UC3); dst_r.copyTo(dst); // iterate through all the top-level contours, // draw each connected component with its own random color int idx = 0; double maxArea = 0.0; int maxId = -1; for(unsigned int i = 0; i<contours.size(); i++) { Mat temp(contours.at(i)); double area = fabs(contourArea(temp)); if(area > maxArea) { maxId = i; maxArea = area; } } //for( ; idx >= 0; idx = hierarchy[idx][0] ) //{ // //Scalar color( rand()&255, rand()&255, rand()&255 ); // //drawContours(dst, contours, idx, color, CV_FILLED, 8, hierarchy ); // double area = contourArea(contours.at(idx)); // if(area > maxArea) // { // maxId = idx; // maxArea = area; // } //} //显示最大轮廓外形，以及最佳匹配的模板ID if(contours.size() > 0) { Scalar color(0, 255, 255 ); drawContours(dst, contours, maxId, color); Mat hand(contours.at(maxId)); int value = hand_template_match(hand); if(value >= 0) { Scalar templateColor(255, 0, 255 ); drawContours(dst, g_TemplateContours, value, templateColor); printf("Match %d /r/n", value); stringstream ss; ss << "Match " << value; string text = ss.str(); putText(dst, text, Point(300, 30), FONT_HERSHEY_SIMPLEX, 1.0, templateColor); } } } unsigned short GetNearestPointByUser(const xn::DepthMetaData& dmd, const xn::SceneMetaData& smd) { XnUInt16 g_nXRes = dmd.XRes(); XnUInt16 g_nYRes = dmd.YRes(); unsigned short minDepth = 10000; for (XnUInt16 nY=0; nY<g_nYRes; nY++) { for (XnUInt16 nX=0; nX<g_nXRes; nX++) { if( (smd(nX,nY) != 0) && (dmd(nX,nY) < minDepth)) minDepth = dmd(nX,nY); //if( (smd(nY,nX) != 0) && (dmd(nY,nX) < minDepth)) // minDepth = dmd(nY, nX); } } return minDepth; } void GetHandByUser(Mat& hand, unsigned short nearest, const xn::DepthMetaData& dmd, const xn::SceneMetaData& smd) { XnUInt16 g_nXRes = dmd.XRes(); XnUInt16 g_nYRes = dmd.YRes(); Mat dst = Mat::zeros(g_nYRes, g_nXRes, CV_8UC1); dst.copyTo(hand); for (XnUInt16 nY=0; nY<g_nYRes; nY++) { for (XnUInt16 nX=0; nX<g_nXRes; nX++) { if( (smd(nX,nY) != 0) && (dmd(nX,nY) < (nearest + g_backwardDist) ) ) { hand.at<unsigned char>(nY,nX) = 0xFF; //uchar *p = hand.data + nY * g_nXRes + nX; //*p = 0xFF; } } } } int HandDetect() { char key=0; init_hand_template(); XnStatus eResult = XN_STATUS_OK; //initial val xn::DepthMetaData m_DepthMD; //xn::ImageMetaData m_ImageMD; xn::SceneMetaData m_sceneMD; // for opencv Mat //Mat m_depth16u( 480,640,CV_16UC1); //Mat m_user16u( 480,640,CV_16UC1); //Mat m_rgb8u( 480,640,CV_8UC3); Mat m_DepthShow( 480,640,CV_8UC1); Mat m_UserShow( 480,640,CV_8UC1); //Mat m_ImageShow( 480,640,CV_8UC3); Mat m_DepthThreshShow( 480,640,CV_8UC1); Mat m_HandShow( 480,640,CV_8UC3); //initial context xn::Context mContext; eResult = mContext.Init(); //xn::EnumerationErrors errors; //eResult = mContext.InitFromXmlFile(SAMPLE_XML_PATH, &errors); CheckOpenNIError( eResult, "initialize context" ); //Set mirror mContext.SetGlobalMirror(!mContext.GetGlobalMirror()); //create depth generator xn::DepthGenerator mDepthGenerator; eResult = mDepthGenerator.Create( mContext ); CheckOpenNIError( eResult, "Create depth generator" ); //create image generator xn::ImageGenerator mImageGenerator; eResult = mImageGenerator.Create( mContext ); CheckOpenNIError( eResult, "Create image generator" ); //set map mode XnMapOutputMode mapMode; mapMode.nXRes = 640; mapMode.nYRes = 480; mapMode.nFPS = 30; eResult = mDepthGenerator.SetMapOutputMode( mapMode ); eResult = mImageGenerator.SetMapOutputMode( mapMode ); //Create user generator xn::UserGenerator mUserGenerator; eResult = mUserGenerator.Create( mContext ); CheckOpenNIError( eResult, "Create user generator" ); //Set callback by option //XnCallbackHandle hUserCallbacks; //mUserGenerator.RegisterUserCallbacks(User_NewUser, User_LostUser, NULL, hUserCallbacks); //由于 Kinect 的深度摄像机和彩色摄像机是在不同的位置，而且镜头本身的参数也不完全相同，所以两个摄像机所取得的画面会有些微的差异 //将深度摄像机的视角调整到RGB摄像机位置 // correct view port //mDepthGenerator.GetAlternativeViewPointCap().SetViewPoint( mImageGenerator ); //start generate data eResult = mContext.StartGeneratingAll(); CheckOpenNIError( eResult, "Start generate" ); //read data eResult = mContext.WaitNoneUpdateAll(); while( (key!=27) && !(eResult = mContext.WaitNoneUpdateAll( )) ) { //get the depth map mDepthGenerator.GetMetaData(m_DepthMD); //memcpy(m_depth16u.data,m_DepthMD.Data(), 640*480*2); //get the user map mUserGenerator.GetUserPixels(0, m_sceneMD); //memcpy(m_user16u.data,m_sceneMD.Data(), 640*480*2); //get the image map //mImageGenerator.GetMetaData(m_ImageMD); //memcpy(m_rgb8u.data,m_ImageMD.Data(),640*480*3); //Get nearest point unsigned short nearest = GetNearestPointByUser(m_DepthMD, m_sceneMD); //printf("Nearest is %d/n", nearest); //Get user hand GetHandByUser(m_DepthShow, nearest, m_DepthMD, m_sceneMD); //在此对灰度图像进行处理，平滑和去噪声 //erode(m_DepthShow, m_DepthThreshShow, Mat(), Point(-1,-1), 3); //dilate(m_DepthThreshShow, m_DepthShow, Mat(), Point(-1,-1), 3); medianBlur(m_DepthShow, m_DepthThreshShow, 3); //m_DepthThreshShow.copyTo(m_DepthShow); medianBlur(m_DepthThreshShow, m_DepthShow, 3); //blur(m_DepthShow, m_DepthThreshShow, Size(3, 3)); //m_DepthThreshShow.copyTo(m_DepthShow); //blur(m_DepthThreshShow, m_DepthShow, Size(3, 3)); //Mat pyrTemp( 240,320,CV_8UC1); //pyrDown(m_DepthShow, pyrTemp); //pyrUp(pyrTemp, m_DepthShow); imshow( "Hand", m_DepthShow ); findHand(m_DepthShow, m_HandShow); imshow( "Result", m_HandShow ); key=cvWaitKey(20); } // 10. stop mContext.StopGeneratingAll(); mContext.Shutdown(); return 0; } int _tmain(int argc, _TCHAR* argv[]) { HandDetect(); }