手势虚拟键盘
定义一个HandDetector类
import cv2 import mediapipe as mp import math class HandDetector: """ Finds Hands using the mediapipe library. Exports the landmarks in pixel format. Adds extra functionalities like finding how many fingers are up or the distance between two fingers. Also provides bounding box info of the hand found. """ def __init__(self, mode=False, maxHands=2, detectionCon=0.5, minTrackCon=0.5): """ :param mode: In static mode, detection is done on each image: slower :param maxHands: Maximum number of hands to detect :param detectionCon: Minimum Detection Confidence Threshold :param minTrackCon: Minimum Tracking Confidence Threshold """ self.mode = mode self.maxHands = maxHands self.detectionCon = detectionCon self.minTrackCon = minTrackCon self.mpHands = mp.solutions.hands self.hands = self.mpHands.Hands(static_image_mode=self.mode, max_num_hands=self.maxHands, min_detection_confidence=self.detectionCon, min_tracking_confidence = self.minTrackCon) self.mpDraw = mp.solutions.drawing_utils self.tipIds = [4, 8, 12, 16, 20] self.fingers = [] self.lmList = [] def findPosition(self, img, draw=True): self.lmList = [] if self.results.multi_hand_landmarks: for handLms in self.results.multi_hand_landmarks: for id, lm in enumerate(handLms.landmark): h, w, c = img.shape cx, cy = int(lm.x * w), int(lm.y * h) # print(id, cx, cy) self.lmList.append([id, cx, cy]) if draw: cv2.circle(img, (cx, cy), 12, (255, 0, 255), cv2.FILLED) return self.lmList def findHands(self, img, draw=True, flipType=True): """ Finds hands in a BGR image. :param img: Image to find the hands in. :param draw: Flag to draw the output on the image. :return: Image with or without drawings """ imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) self.results = self.hands.process(imgRGB) allHands = [] h, w, c = img.shape if self.results.multi_hand_landmarks: for handType,handLms in zip(self.results.multi_handedness,self.results.multi_hand_landmarks): myHand={} ## lmList mylmList = [] xList = [] yList = [] for id, lm in enumerate(handLms.landmark): px, py = int(lm.x * w), int(lm.y * h) mylmList.append([px, py]) xList.append(px) yList.append(py) ## bbox xmin, xmax = min(xList), max(xList) ymin, ymax = min(yList), max(yList) boxW, boxH = xmax - xmin, ymax - ymin bbox = xmin, ymin, boxW, boxH cx, cy = bbox[0] + (bbox[2] // 2), \ bbox[1] + (bbox[3] // 2) myHand["lmList"] = mylmList myHand["bbox"] = bbox myHand["center"] = (cx, cy) if flipType: if handType.classification[0].label =="Right": myHand["type"] = "Left" else: myHand["type"] = "Right" else:myHand["type"] = handType.classification[0].label allHands.append(myHand) ## draw if draw: self.mpDraw.draw_landmarks(img, handLms, self.mpHands.HAND_CONNECTIONS) cv2.rectangle(img, (bbox[0] - 20, bbox[1] - 20), (bbox[0] + bbox[2] + 20, bbox[1] + bbox[3] + 20), (255, 0, 255), 2) cv2.putText(img,myHand["type"],(bbox[0] - 30, bbox[1] - 30),cv2.FONT_HERSHEY_PLAIN, 2,(255, 0, 255),2) if draw: return allHands,img else: return allHands def fingersUp(self,myHand): """ Finds how many fingers are open and returns in a list. Considers left and right hands separately :return: List of which fingers are up """ myHandType =myHand["type"] myLmList = myHand["lmList"] if self.results.multi_hand_landmarks: fingers = [] # Thumb if myHandType == "Right": if myLmList[self.tipIds[0]][0] > myLmList[self.tipIds[0] - 1][0]: fingers.append(1) else: fingers.append(0) else: if myLmList[self.tipIds[0]][0] < myLmList[self.tipIds[0] - 1][0]: fingers.append(1) else: fingers.append(0) # 4 Fingers for id in range(1, 5): if myLmList[self.tipIds[id]][1] < myLmList[self.tipIds[id] - 2][1]: fingers.append(1) else: fingers.append(0) return fingers def findDistance(self,p1, p2, img=None): """ Find the distance between two landmarks based on their index numbers. :param p1: Point1 :param p2: Point2 :param img: Image to draw on. :param draw: Flag to draw the output on the image. :return: Distance between the points Image with output drawn Line information """ x1, y1 = p1 x2, y2 = p2 cx, cy = (x1 + x2) // 2, (y1 + y2) // 2 length = math.hypot(x2 - x1, y2 - y1) info = (x1, y1, x2, y2, cx, cy) if img is not None: cv2.circle(img, (x1, y1), 15, (255, 0, 255), cv2.FILLED) cv2.circle(img, (x2, y2), 15, (255, 0, 255), cv2.FILLED) cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), 3) cv2.circle(img, (cx, cy), 15, (255, 0, 255), cv2.FILLED) return length,info, img else: return length, info def main(): cap = cv2.VideoCapture(0) detector = HandDetector(detectionCon=0.8, maxHands=2) while True: # Get image frame success, img = cap.read() # Find the hand and its landmarks hands, img = detector.findHands(img) # with draw # hands = detector.findHands(img, draw=False) # without draw if hands: # Hand 1 hand1 = hands[0] lmList1 = hand1["lmList"] # List of 21 Landmark points bbox1 = hand1["bbox"] # Bounding box info x,y,w,h centerPoint1 = hand1['center'] # center of the hand cx,cy handType1 = hand1["type"] # Handtype Left or Right fingers1 = detector.fingersUp(hand1) if len(hands) == 2: # Hand 2 hand2 = hands[1] lmList2 = hand2["lmList"] # List of 21 Landmark points bbox2 = hand2["bbox"] # Bounding box info x,y,w,h centerPoint2 = hand2['center'] # center of the hand cx,cy handType2 = hand2["type"] # Hand Type "Left" or "Right" fingers2 = detector.fingersUp(hand2) # Find Distance between two Landmarks. Could be same hand or different hands length, info, img = detector.findDistance(lmList1[8], lmList2[8], img) # with draw # length, info = detector.findDistance(lmList1[8], lmList2[8]) # with draw # Display cv2.imshow("Image", img) cv2.waitKey(1) if __name__ == "__main__": main()
虚拟输入
import cv2 from cvzone.HandTrackingModule import HandDetector from time import sleep import pyautogui import cvzone from pynput.keyboard import Key,Controller cap = cv2.VideoCapture(0) cap.set(3, 1280) cap.set(4, 720) # 识别手势 detector = HandDetector(detectionCon=0.8) keyboard = Controller() # 键盘关键字 keys = [['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'], ['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', ';'], ['Z', 'X', 'C', 'V', 'B', 'N', 'M', ',', '.', '/']] class Button(): def __init__(self, pos, text, size = [50, 50]): self.pos = pos self.text = text self.size = size buttonList = [] finalText = '' for j in range(len(keys)): for x, key in enumerate(keys[j]): # 循环创建buttonList对象列表 buttonList.append(Button([60*x+20,100+j*60],key)) def drawAll(img,buttonList): for button in buttonList: x, y = button.pos w, h = button.size cvzone.cornerRect(img, (x, y, w, h),20,rt = 0) cv2.rectangle(img, button.pos, (x + w, y + h), (255, 0, 255), cv2.FILLED) cv2.putText(img, button.text, (x + 10, y + 40), cv2.FONT_HERSHEY_PLAIN, 3, (255, 255, 255), 2) return img while True: success, img = cap.read() # 识别手势 hand = detector.findHands(img) lmList = detector.findPosition(img, True) # img = mybutton.draw(img) img = drawAll(img,buttonList ) if lmList: for button in buttonList: x,y = button.pos w,h = button.size if x
最后：代码我完全开源可用，需要安装必要的python包。