OpenCV+Mediapipe+UDP+Unity挥手电子书翻页

OpenCV+Mediapipe+UDP+Unity挥手电子书翻页

效果视频如下

OpenCV+Mediapipe+UDP+Unity挥手翻页

Python端

其实很简单，基本都是现成的东西。
主要做的事情：

1. 读取XML配置文件，主要是摄像头的配置（包括分辨率、帧率）、网络配置、调试信息的配置等。
2. Opencv读取摄像头，Mediapipe识别手，UDP发送识别到的手坐标数据。
3. PyInstaller将py代码打包为EXE。

   这里有个坑：
   pyinstaller打包的exe，总提示找不到mediapipe路径，需要打包时手工加入mediapipe库：

   pyinstaller --add-data="E:/MyPythonProjects/HandTouch/venv/Lib/site-packages/mediapipe/modules;mediapipe/modules" -w main.py
   

   第二个坑：打包完的Exe路径不能含有中文字符，否则还是提示找不到路径。

关键代码段也就这么些：

cap = cv2.VideoCapture(usbpt)
if width > 0 and height > 0:
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
if vfps > 0:
    cap.set(cv2.CAP_PROP_FPS, vfps)

mpHands = mp.solutions.hands
hands = mpHands.Hands(max_num_hands=1)
mpDraw = mp.solutions.drawing_utils

pTime = 0

while True:
    success, img = cap.read()
    if success:
        if bFilp:
            img = cv2.flip(img, 1)
        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        h, w, t = img.shape
        result = hands.process(imgRGB)
        if result.multi_hand_landmarks:
            lm = result.multi_hand_landmarks[0]
            hd = lm.landmark[0]
            x = int(hd.x * w)
            y = int(hd.y * h)
            if bShowHand:
                mpDraw.draw_landmarks(img, lm, mpHands.HAND_CONNECTIONS)
                cv2.circle(img, (x, y), 3, (0, 255, 255), 3)
            skt.sendto(('[%d,%d]' % (x, y)).encode('utf-8'), (host, port))

        if bShowFps:
            cTime = time.time()
            fps = int(1 / (cTime - pTime))
            pTime = cTime
            cv2.putText(img, str(int(fps)), (20, 60), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3)

        if bShowImg:
            cv2.imshow("Image", img)

        if bEscQuit and (cv2.waitKey(1) & 0xff == 27):
            break
        else:
            cv2.waitKey(1)

cv2.destroyAllWindows()
cap.release()

Unity端

数据收到后，有穷自动状态机，解析一下数据：

// 数据接收线程
private void ReceiveFrom()
{
    EndPoint remote = new IPEndPoint(IPAddress.Any, 0);
    byte[] data = new byte[1024];

    while (IsStart)
    {
        try
        {
            int len = m_socket.ReceiveFrom(data, data.Length, SocketFlags.None, ref remote);
            for (int i = 0; i < len; ++i)
            {
                switch (m_state)
                {
                    case ReceState.START:
                        if (data[i] == '[')
                        {
                            m_state = ReceState.XDATA;
                            m_pos.x = 0;
                        }
                        break;

                    case ReceState.XDATA:
                        if (data[i] >= '0' && data[i] <= '9')
                            m_pos.x = m_pos.x * 10 + (data[i] - '0');
                        else if (data[i] == ',')
                        {
                            m_state = ReceState.YDATA;
                            m_pos.y = 0;
                        }
                        else
                            m_state = ReceState.START;
                        break;

                    case ReceState.YDATA:
                        if (data[i] >= '0' && data[i] <= '9')
                            m_pos.y = m_pos.y * 10 + (data[i] - '0');
                        else if (data[i] == ']')
                        {
                            thePosition.Enqueue(m_pos);
                            m_state = ReceState.START;
                        }
                        else
                            m_state = ReceState.START;
                        break;
                }
            }
        }
        catch { }
    }
}

对于收到的数据，进一步解析，获得手势：

这里只是用了很简单的判断方式：

1. 只判断水平方向的挥手
2. 只看累计位移量（像素）是否达到阈值

private void OnReceivePos(Vector2 pos)
{
    if( bCanTrigger )
    {
        waitTime = 0.333f;
        if (posCount++ > 0)
        {
            float delta = pos.x - lastPos;
            totalDelta += delta;
            if (posCount > 5 )
            {
                if (totalDelta > flipValue)
                {
                    bCanTrigger = false;
                    interval = autoFlip.PageFlipTime;
                    lastPos = 0;
                    totalDelta = 0;
                    posCount = 0;
                    PrevPage();
                }
                else if (totalDelta < -flipValue)
                {
                    bCanTrigger = false;
                    interval = autoFlip.PageFlipTime;
                    lastPos = 0;
                    totalDelta = 0;
                    posCount = 0;
                    NextPage();
                }
            }
        }
        lastPos = pos.x;
    }
}