基于单 camera的手势识别

android平台测试程序见附件。

http://pan.baidu.com/s/1gd5nthL    

密码： h46g

基于视觉的动作识别，一直以来的最大问题是精度问题。 - -！

刚刚做了一套手势识别的算法，在此做下简单总结

先说下我做的效果：

     2m内手掌识别率在90%以上（强光干扰下效果烂成渣渣了。。so。。 扣掉10%）

     处理速度，15fps 

     cpu  30%  （arm 4核）

以上效果为平板测试结果

1> 目的： opencv的haar特征库用来训练各种手势其实是很强势，很好用的东西，唯一的缺点是： Haar版权问题。。，

     Haar训练需要的样本数也是一个蛮头疼的问题，识别准确性完全取决于样本数量和质量，没有几十k的样本，效果只能呵呵了

    还好，条条大路通罗马，

2>外围设备：  摄像头，这是基本的

                          深度传感器     ，这是一直想要的，可惜到现在也没搞到

                         红外传感器        ，同上。。。。。

3> 检测方式：  利用手势的特征点：

                         手掌的特征点还是挺多的，我使用的是手轮廓的5个内凹陷，再加上相对位置，再加上人体肤色的特征

                         基本一个完整的手就检测出来了

4> 缺点：   精度缺失 ，没办法识别3d空间位置，单camera。。。。

--------------------------------code 分割线--------------------------------------------------------

肤色过滤部分

#include "utils.h"

cv::Mat edgeFilter();
void areaFilter(cv::Mat srcMask);
void skinFilter();


cv::Scalar YUV_SKIN_BEGIN = cv::Scalar(0,133,77);   // 论文肤色(0,133,77)->(256,173,127)
cv::Scalar YUV_SKIN_END   = cv::Scalar(256,173,127);

cv::Scalar COLOR_RED = cv::Scalar(0,0,255);
cv::Scalar COLOR_GREEN = cv::Scalar(0,255,0);
cv::Scalar COLOR_BLUE = cv::Scalar(255,0,0);

#ifdef USE_SHARP
cv::Mat sharpKernal = (cv::Mat_(3, 3) << 0, -1, 0, -1, 5, -1, 0, -1, 0);
#endif

cv::Mat morphKernal = cv::getStructuringElement(cv::MORPH_RECT,cv::Size(3,3),cv::Point(1,1) );

cv::Size pSize = cv::Size(320,240);
//  边缘轮廓
cv::Mat edgeFilter(){
#ifdef USE_EDGE
#ifdef DEBUG
long begT = getTime(true);
#endif
    cv::Mat g = gray.clone();

    if(withRsize)
        resize(g,g,pSize);
    equalizeHist(g,g);
    Canny(g,g,100,300,3);
    g = g > 1;

#ifdef USE_SHARP
long usec = getTime(true);
    filter2D(gray, gray, gray.depth(), sharpKernal); //卷积
LOGD("filter2D(%d*%d): %d us",gray.cols,gray.rows,(int)(getTime(true)-usec));
#endif

#ifdef USE_ERODE
    dilate(gray,gray,morphKernal);
#endif

    if(withRsize)
        resize(g,g,src.size());

#ifdef DEBUG
    int w = withRsize? 320 : gray.cols;
    int h = withRsize? 240 : gray.rows;
    cv::Mat can;
    cvtColor(g,can,CV_GRAY2BGR);
    //src.copyTo(can,g);
    //addWeighted(src,1.0,can,1.0,1.0,src);
    src = src + can;
LOGD("edgeFilter(%d*%d): %d us",w,h,(int)(getTime(true)-begT));
#endif
    return g;
#endif //USE_EDGE
}

// 小面积过滤
void areaFilter(cv::Mat srcMask){
#ifdef USE_SF_AREA
#ifdef DEBUG
long usec = getTime(true);
#endif
    cv::Mat temp = srcMask.clone(); // 二值化图像
    std::vector > points;
    std::vector vecs;
    findContours(temp, points, vecs,
	      CV_RETR_EXTERNAL,  CV_CHAIN_APPROX_SIMPLE );

    if(points.size()<=0 || vecs.size()<=0) {
         skin.setTo(0);
         return;
    }

    std::vector > points_poly(points.size());
    std::vector bound(points.size());
    std::vector usefulList;
    std::vector areaList;
    float max=0;
    for(int i=0;i>=0;i=vecs[i][0]){
	 bound[i] = boundingRect(points[i]);
         //噪声过滤（微小面积过滤）
         if( points[i].size()<6
        	|| bound[i].width < MIN_IMAGE_PIX || bound[i].height < MIN_IMAGE_PIX
                ) {
            srcMask(bound[i]).setTo(0);
            continue;
         }else{ // 多边形逼近
            approxPolyDP(points[i], points_poly[i],10,true);
            float area = fabs(contourArea(points_poly[i]));
            // 有用Rect
            usefulList.push_back(i);  // 索引
            areaList.push_back(area); // 面积
            if(max=0;j--){
        i = usefulList.at(j);
        float area = areaList.at(j);
        //过滤
        if(area<=max) {
           srcMask(bound[i]).setTo(0);
           continue;
        }
#ifdef USE_ERODE
        std::vector hull;
        std::vector pos = points_poly[i];
        convexHull(cv::Mat(pos), hull, true);
        try{
            std::vector defects; // 起始点，终止点，最凹点，凹点深度
            convexityDefects(pos,cv::Mat(hull),defects);
            int dsize = defects.size();
            if(dsize<=0) continue;
            int miniDis = MAX(bound[i].width,bound[i].height)>>3;

            for(int j=0;j(defects[j][3]>>8)) continue;
               cv::Rect rect = cv::Rect(pos[defects[j][2]].x-(miniDis>>1)
            		   ,pos[defects[j][2]].y-(miniDis>>1)
            		   ,(miniDis)
            		   ,(miniDis));
               cv::Mat roi = srcMask(rect);
               cv::Mat kernel = getStructuringElement(cv::MORPH_RECT,cv::Size(3,(miniDis>>1)+1),cv::Point(1,(miniDis>>2)+1) );
               erode(roi,roi,kernel);
               medianBlur(roi,roi,3);
            }

         }catch(cv::Exception e){}
#endif
    }
#ifdef DEBUG
LOGD("areaFilter(%d*%d): %d us",srcMask.cols,srcMask.rows,(int)(getTime(true)-usec));
#endif
#endif
}

cv::Mat skin_img;
cv::Mat cany;
void *edgeFilterT(void*){
    skin_img = edgeFilter();
}

void skinFilter(){
#ifdef DEBUG
long usec = getTime(true);
LOGD("skinFilter()");
#endif

#ifdef USE_EDGE
int err;
pthread_t tid;
err = pthread_create(&tid,NULL,&edgeFilterT,NULL);
if(err!=0){
	skin_img = edgeFilter();
}
#endif

     cv::Mat tmpColor;
     cv::cvtColor(src,tmpColor,CV_BGR2YCrCb);
     cv::Mat skinMask = cv::Mat::zeros(tmpColor.size(),CV_8UC1);
     cv::inRange(tmpColor,YUV_SKIN_BEGIN,YUV_SKIN_END,skinMask);

#ifdef USE_EDGE
    if(err==0){
        err = pthread_join(tid,NULL);
    }
     skinMask -= skin_img;
     cany = skin_img;
#endif
     skinMask -= gray;

#ifdef USE_SF_AREA
    areaFilter(skinMask);
#endif

#ifdef DEBUG
LOGD("skinFilter:BGR->YUV:(%d,%d) : %d us",src.cols,src.rows,(int)(getTime(true)-usec));
#endif
     mask = skinMask;    // yuv uv分量肤色mask
}

手掌识别部分：

#include "utils.h"

// (0 - 3)   0:最准确  3:误判最多
#define PLAM_THRESHOLD 0
bool isPlamFound = false;
bool withPlam = true;

void actionPlam(RectHold found);
void actionFist(cv::Point center);
void findArea(); // 面积处理
bool findPlam(cv::Mat mask,cv::Rect roi);
int getDistance(cv::Point p1,cv::Point p2,bool usemax=false);
//void findPlam(std::vector uContours);  // 计算轮廓

int getDistance(cv::Point p1,cv::Point p2,bool usemax) {
   int x = abs(p1.x-p2.x);
   int y = abs(p1.y-p2.y);
   return usemax ? MAX(x,y) : sqrt(x*x+y*y);
}

void actionPlam(RectHold found){
//#ifdef DEBUG
	if(showRect)
        rectangle(src,found.bound.tl(),found.bound.br(),COLOR_RED,3,8,0);
	LOGD("found plam (%d,%d) ,%d",found.center.x,found.center.y,found.c);
//#endif
    cv::Rect pRect = found.bound;
    //lastPlamHold.lastRect = cv::Rect(pRect.tl(),pRect.br());
    lastPlamHold.lastRect = found.bound;
    lastPlamHold.center = found.center;
    lastPlamHold.area = found.area;
    lastPlamHold.arc = found.arc;
   // lastPlamHold.centRect = cv::Rect(found.center.x-(pRect.width>>2)
   //        ,found.center.y-(pRect.height>>2) ,pRect.width>>1 ,pRect.height>>1);
    lastPlamHold.centRect = found.cRect;
    if(!lastPlamHold.used){
        lastPlamHold.used = true;
        lastPlamHold.unHoldCount=0;
        return;
    }
#ifdef USE_OPT_FLOW
     optInit = true;

#ifndef OPT_NEED_INIT
     withPlam = false;
#endif

#endif
     // 关闭camshift
#ifdef USE_CAMSHIFT
#ifdef USE_FIST
     trackRect = false;
     initBackproj = false;
#else
     trackRect = true;
     initBackproj = true;
#endif // USE_FIST
#endif // USE_CAMSHIFT
   /////////////////
    lastPlamHold.type = TYPE_PLAM;
    callJava(lastPlamHold.type,lastPlamHold.center);
}

void actionFist(cv::Point center){
     lastPlamHold.used = true;
     lastPlamHold.unHoldCount=0;
     lastPlamHold.center = center;

     // 关闭camshift
#ifdef USE_CAMSHIFT
     trackRect = true;
     initBackproj = true;
#endif // USE_CAMSHIFT
   /////////////////
    lastPlamHold.type = TYPE_FIST;
    callJava(lastPlamHold.type,lastPlamHold.center);
}

cv::Mat plam_img;
cv::Rect plam_rect;
pthread_mutex_t locker = PTHREAD_MUTEX_INITIALIZER;
void *findPlamT(void*){
    findPlam(plam_img,plam_rect);
}

//   截取不同区域计算手掌
long pTime = 0;
std::vector pHolds;
bool findPlam(cv::Mat mask,cv::Rect roi){
#ifdef DEBUG
long beginT = getTime(true);
#endif
       cv::Point sp = cv::Point(roi.x,roi.y);
       cv::Rect   bound_find;
       std::vector  contours_find;
       std::vector  contours_poly_find;
       std::vector plamHold;
       float max = -1;
       float arc = -1;
{
       int width = roi.width >> 1;
       int height = roi.height >> 1;
       bool wLarge = width >= height;
       int mMax = MAX(roi.width,roi.height);
       int mMin = MIN(roi.width,roi.height);
       if(mMin<=8) return false;
       int sv = mMax/mMin;
       if(sv>1 && mMax>30){
       //if(false) {
           // width : height not normal split it
    	   cv::Rect roiRect1,roiRect2;  // 截取两端
    	   int x,y,w,h;
                if(wLarge){  // (0,0,1.5w,1.5h)
                	// for roiRect1
                    w = roi.height + height - (height>>2);
                    h = roi.height;
                    // for roiRect2
                    x = roi.x+roi.width-w;
                    y = roi.y;

                }else{
                    w = roi.width;
                    h = roi.width + width - (width>>2);
                    x = roi.x;
                    y = roi.y+roi.height-h;
                }

                roiRect1 = cv::Rect(roi.x,roi.y,w,h) & cv::Rect(0,0,mask.cols,mask.rows);
                roiRect2 = cv::Rect(x,y,w,h) & cv::Rect(0,0,mask.cols,mask.rows);

                plam_img = mask;
                plam_rect = roiRect1;
#ifdef USE_THREAD
                pthread_t tid;
                pthread_create(&tid,NULL,&findPlamT,NULL);
#else
                findPlam(mask,roiRect1);// recursive
#endif
                findPlam(mask,roiRect2);// recursive
#ifdef USE_THREAD
                pthread_join(tid,NULL);
#endif
           return false;
       }
       cv::Mat tmp = mask(roi).clone();
       std::vector > contours;
       std::vector vecs;
       findContours(tmp,contours,vecs,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE);

       if(contours.size()<=0 || vecs.size()<=0)    return false;

       std::vector > contours_poly(contours.size());
       std::vector bound(contours.size());

       for(int i=0;i>=0;i=vecs[i][0]){
           bound[i] = boundingRect(contours[i]);
           if(contours[i].size()>6
                && bound[i].width>width
                && bound[i].height>height){
               approxPolyDP(contours[i],contours_poly[i],5.0,true);
               if(contours_poly[i].size()<=3) continue;
               float area = fabs(contourArea(contours_poly[i]));
               if(area>max){
            	   arc = arcLength(contours_poly[i],true);
                   max  = area;
                   bound_find = bound[i];
                   contours_find = contours[i];
                   contours_poly_find = contours_poly[i];
               }
           }
       } // for end
}
     if(contours_poly_find.size()<=3) return false;
     cv::Rect shRect = cv::Rect(bound_find.x+roi.x,bound_find.y+roi.y,bound_find.width,bound_find.height);
     // now we find the max area,let's find out the plam

     //Moments m = moments(contours_find,false);
     // 绘制 凸包
     cv::Rect cRect = cv::Rect(bound_find.x + (bound_find.width>>2)
               ,bound_find.y + (bound_find.height>>2)
               ,bound_find.width>>1
               ,bound_find.height>>1);

     cv::Point center = cv::Point(cRect.x+(cRect.width>>1),cRect.y+(cRect.height>>1))+sp;
     /*
     if(lastPlamHold.used && lastPlamHold.centRect.contains(center)){
    	 rectangle(src,cRect.tl()+sp,cRect.br()+sp,COLOR_RED,3,8,0);
    	 isPlamFound = true;
     }
     */
     //rectangle(src,cRect.tl()+sp,cRect.br()+sp,COLOR_RED,0.5,8,0);
     std::vector hull;
     std::vector pos = contours_poly_find;
     convexHull(cv::Mat(pos), hull, true);
     try{
         std::vector defects; // 起始点，终止点，最凹点，凹点深度
         convexityDefects(pos,cv::Mat(hull),defects);
         int dsize = defects.size();
         if(dsize<=0) return false;
         // 凸凹陷点均值
         int avg=0;
         for(int j=0;j>8);
         avg /= dsize;
         int miniDis = avg>>1;

         cv::Point fixP = cv::Point(0,0); // 偏移修正量
         int actSize = 0;
         std::vector pHold;
         for(int j=0;j>8) <= avg){
                 continue;
            }
            pHold.push_back(j);
            cv::Point depP = pos[defects[j][2]]; //凸凹陷点
            // cRect = 1/2 Bound.Rect
            // 向落入cRect的极值点方向偏移，偏移量 fixP(x,y)
            if(cRect.contains(pos[defects[j][2]])){ // 蓝色 一次修正
#ifdef DEBUG
                 if(showRect){
                    circle(src,pos[defects[j][2]]+sp,4,COLOR_BLUE,0.5,CV_AA);
                    line(src,pos[defects[j][1]]+sp,pos[defects[j][0]]+sp,COLOR_BLUE,2,CV_AA);
                 }
#endif
                 fixP.x += depP.x;
                 fixP.y += depP.y;
                 actSize ++;
            }
#ifdef DEBUG
            if(showRect){
                  cv::Vec4i vec = defects[j];
                  line(src,pos[vec[0]]+sp,pos[vec[2]]+sp,COLOR_RED,1.5,CV_AA);
	              line(src,pos[vec[1]]+sp,pos[vec[2]]+sp,COLOR_RED,2,CV_AA);
		          // farthest_pt_index
                  circle(src,pos[vec[2]]+sp,4,COLOR_RED,1,CV_AA);
                  circle(src,pos[vec[1]]+sp,4,COLOR_BLUE,2,CV_AA);
                  circle(src,pos[vec[0]]+sp,4,COLOR_GREEN,1,CV_AA);
            } // end showRect
#endif
         }//end for defects.size();
             if(actSize<=1) {
#ifdef USE_NEW_FIST
            	 // ==========================maybe fist
            	 int areaBit = (int)lastPlamHold.area/max;

            	 //int pMax = MAX(abs(lastPlamHold.center.x-center.x),abs(lastPlamHold.center.y-center.y));
            	 if(lastPlamHold.used&&lastPlamHold.centRect.contains(center)
            			 && lastPlamHold.lastRect.width > shRect.width
            			 && lastPlamHold.lastRect.height > shRect.width
            			 //&& pMax>2
            			 && areaBit>=0 && areaBit<3
            			 //&& pHold.size()<=2
            			 ){

            		 rectangle(src,shRect.tl(),shRect.br(),COLOR_BLUE,3,8,0);
            		 actionFist(center);
            	 }
            	 if(lastPlamHold.unHoldCount++>fps) lastPlamHold.used=false;
        initBackproj = true;
        sRect = cRect & cv::Rect(0,0,src.cols,src.rows);
#endif
            	 return false;   // fixP 偏移修正(x,y)
             }
             fixP.x /= actSize;
             fixP.y /= actSize;
             fixP.x -= (cRect.x + (cRect.width>>1));   // fixP 中心偏移
             fixP.y -= (cRect.y + (cRect.height>>1));
             // 修正后的rect
           int lx = MIN(cRect.width,cRect.height)>>3;
           cv::Rect actRect = cv::Rect(cRect.x+fixP.x-lx,cRect.y+fixP.y-lx
                            ,cRect.width+(lx<<1),cRect.height+(lx<<1));
#ifdef DEBUG
           rectangle(src,actRect.tl()+sp,actRect.br()+sp,COLOR_RED,0.5,8,0);
#endif

           std::vector tmpHold;
           for(int i=pHold.size()-1;i>=0;i--){
                cv::Point p = pos[defects[pHold[i]][2]];
                if(actRect.contains(p))
                	tmpHold.push_back(pHold[i]);  // 生效点
           }
           // 判断极值点是否过近
           for(int i=tmpHold.size()-1;i>0;i--){
        	   cv::Point p1 = pos[defects[tmpHold[i]][2]];
        	   cv::Point p2 = pos[defects[tmpHold[i-1]][2]];
        	   int x = MAX(p1.x,p2.x)-MIN(p1.x,p2.x);
        	   int y = MAX(p1.y,p2.y)-MIN(p1.y,p2.y);
        	   if(MAX(x,y)<=(lx)) {
        		   return false;
        	   }
           }

           int actP=0; // 生效点个数
           // 判断极值点是否与左右相连
           dsize = tmpHold.size();
           for(int i=0;i=(3-PLAM_THRESHOLD)){
                //rectangle(src,roi.tl(),roi.br(),COLOR_RED,3,8,0);
                 RectHold hold;
                 hold.bound = roi;
                 hold.cRect = actRect + sp;
                 hold.center.x = hold.cRect.x + (hold.cRect.width>>1);
                 hold.center.y = hold.cRect.y + (hold.cRect.height>>1);
                 hold.c = 1;
                 hold.area = max ;
                 hold.arc = arc ;
                 plamHold.push_back(hold);
            }
     }catch(cv::Exception e){}
    if(plamHold.size()<=0) {
#ifdef DEBUG
  LOGD("findPlam(%d*%d) (no): %d us",roi.width,roi.height,(int)(getTime(true)-beginT));
#endif
        isPlamFound = false;
        // viewpager ass this fist open
//        if(lastPlamHold.unHoldCount++>fps)
//        	lastPlamHold.used=false;
        return false;
    }
#ifdef USE_FPS_PLAM
    actionPlam(plamHold[0]);
#else
    isPlamFound = false;
    //  多帧平均   初始化时间较长
    long t = getTime();
    if(pTime!=t){
        LOGD(" time out....");
    	pTime = t;
    	std::vector tmpHold;
#ifdef USE_THREAD
    	pthread_mutex_lock(&locker);
#endif
    	for(int i=pHolds.size()-1;i>=0;i--){
    	    if(pHolds[i].c>2)
    		    tmpHold.push_back(pHolds[i]);
    	}
    	std::swap(pHolds,tmpHold);
#ifdef USE_THREAD
    	pthread_mutex_unlock(&locker);
#endif
    }

    if(pHolds.size()==0){
#ifdef USE_THREAD
    	pthread_mutex_lock(&locker);
#endif

        std::swap(pHolds,plamHold);

#ifdef USE_THREAD
        pthread_mutex_unlock(&locker);
#endif
        return false;
    }
    for(int i=plamHold.size()-1;i>=0;i--){
        bool unFind = true;
        RectHold ph = plamHold[i];
#ifdef USE_THREAD
        pthread_mutex_lock(&locker);
#endif
    	for(int j=pHolds.size()-1;j>=0;j--){
            RectHold actHold = pHolds[j];
            if(actHold.cRect.contains(ph.center)){
                pHolds[j].c += ph.c;
                unFind = false;
    		    LOGD("__plam hold count: (%d)  fps:%d",pHolds[j].c,fps);
    		    if(pHolds[j].c >= (fps>>2)){
//#ifdef DEBUG
    		    	rectangle(src,actHold.bound,COLOR_RED,3,8,0);
//#endif
                    isPlamFound = true;
                    actHold.c = (fps>>2)+2;
                    pHolds.clear();
                    pHolds.push_back(actHold);
 		            actionPlam(actHold);
 		            pthread_mutex_unlock(&locker);
                    return true;
    		    }
            }
    	}
        if(unFind){
            pHolds.push_back(ph);
        }
#ifdef USE_THREAD
        pthread_mutex_unlock(&locker);
#endif
    }
#endif
#ifdef DEBUG
  LOGD("findPlam(%d*%d): %d us",roi.width,roi.height,(int)(getTime(true)-beginT));
#endif
}

void findArea(){
#ifdef DEBUG
long beginT = getTime(true);
#endif
    //std::vector usefulContoursList;
    cv::Mat temp = mask.clone(); // 二值化图像
    std::vector > points;
    std::vector vecs;
    findContours(temp, points, vecs,
	      CV_RETR_EXTERNAL,  CV_CHAIN_APPROX_SIMPLE );

    if(points.size()<=0 || vecs.size()<=0) {
         skin.setTo(0);
         return;
    }

    std::vector > points_poly(points.size());
    std::vector bound(points.size());
    //std::vector usefulList;
    //std::vector areaList;
    for(int i=0;i>=0;i=vecs[i][0]){
	    bound[i] = boundingRect(points[i]);
         if( points[i].size()<6) {
            mask(bound[i]).setTo(0);
            continue;
         }else{ // 多边形逼近
#ifdef DEBUG
            approxPolyDP(points[i], points_poly[i],10,true);
            if(showRect) drawContours(src,points_poly,i,COLOR_GREEN);
#endif
            /*
            float area = fabs(contourArea(points_poly[i]));
            // 有用Rect
            usefulList.push_back(i);  // 索引
            areaList.push_back(area); // 面积
            */
            if(withPlam)
                findPlam(mask,bound[i]);
        }
    }
    /*
    int j=usefulList.size()-1;
    usefulContoursList.clear();
    for(int i=0;j>=0;j--){
        i = usefulList.at(j);
        float area = areaList.at(j);
          if(showRect) {
	           drawContours(src,points_poly,i,COLOR_GREEN);
          }
          float arc = arcLength(points_poly[i],true);
          // this has some err
          if(arc<=0) continue;
          cv::Moments m = moments(points[i],false);
          if(m.m00<=0) continue;
          UsefulContours   my;
          my.poly = points_poly[i];
         // my.poly = points[i];
          my.bound = bound[i];
          my.ratio = area/arc;
          my.center = cv::Point2f(m.m10/m.m00,m.m01/m.m00);
          usefulContoursList.push_back(my);
          //findPlam(srcMask,bound[i]);
    }
    */
    //////////噪声过滤完毕，开始处理图像
    skin.setTo(0);
    src.copyTo(skin,mask);

#ifdef DEBUG
  addWeighted(src,0.5,skin,0.9,1.0,src);
  LOGD("findArea(%d*%d): %d us",skin.cols,skin.rows,(int)(getTime(true)-beginT));
#endif
#ifdef  PLAM_INIT_ET
  if(!isPlamFound){
	  if(lastPlamHold.unHoldCount++>30){
		  lastPlamHold.unHoldCount = 0;
		  lastPlamHold.used = false;
	  }
  }
#endif
}