基于opencv的AI虚拟画板
提示：以下是本篇文章正文内容，下面案例可供参考
一、opencv实现虚拟画板
0.jpg 
1.jpg 
2.jpg 
3.jpg 
4.jpg 
5.jpg 
6.jpg
二、使用步骤
meidiapip参考：https://google.github.io/mediapipe/solutions/Hands
1.HandTrackingModule.py
手部关键点的分布：
代码如下（示例）：
import cv2
import mediapipe as mp
import math
class HandDetector:
"""
Finds Hands using the mediapipe library. Exports the landmarks
in pixel format. Adds extra functionalities like finding how
many fingers are up or the distance between two fingers. Also
provides bounding box info of the hand found.
"""
def __init__(self, mode=False, maxHands=2, detectionCon=0.5, minTrackCon=0.5):
"""
:param mode: In static mode, detection is done on each image: slower
:param maxHands: Maximum number of hands to detect
:param detectionCon: Minimum Detection Confidence Threshold
:param minTrackCon: Minimum Tracking Confidence Threshold
"""
self.mode = mode
self.maxHands = maxHands
self.detectionCon = detectionCon
self.minTrackCon = minTrackCon
self.mpHands = mp.solutions.hands
self.hands = self.mpHands.Hands(static_image_mode=self.mode, max_num_hands=self.maxHands,
min_detection_confidence=self.detectionCon, min_tracking_confidence = self.minTrackCon)
self.mpDraw = mp.solutions.drawing_utils
self.tipIds = [4, 8, 12, 16, 20]
self.fingers = []
self.lmList = []
def findHands(self, img, draw=True, flipType=True, r=2, t=2, drawRect=True):
"""
Finds hands in a BGR image.
:param img: Image to find the hands in.
:param draw: Flag to draw the output on the image.
:return: Image with or without drawings
"""
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
self.results = self.hands.process(imgRGB)
allHands = []
h, w, _ = img.shape
if self.results.multi_hand_landmarks:
for handType, handLms in zip(self.results.multi_handedness, self.results.multi_hand_landmarks):
myHand={}
## lmList
mylmList = []
xList = []
yList = []
for index, lm in enumerate(handLms.landmark):
px, py = int(lm.x * w), int(lm.y * h)
mylmList.append([index, px, py])
xList.append(px)
yList.append(py)
## bbox
xmin, xmax = min(xList), max(xList)
ymin, ymax = min(yList), max(yList)
boxW, boxH = xmax - xmin, ymax - ymin
bbox = xmin, ymin, boxW, boxH
cx, cy = bbox[0] + (bbox[2] // 2), bbox[1] + (bbox[3] // 2)
myHand["lmList"] = mylmList
myHand["bbox"] = bbox
myHand["center"] = (cx, cy)
# handType
if flipType:
if handType.classification[0].label =="Right":
myHand["type"] = "Left"
else:
myHand["type"] = "Right"
else:
myHand["type"] = handType.classification[0].label
allHands.append(myHand)
## draw
if draw:
if drawRect:
cv2.rectangle(img, (bbox[0] - 20, bbox[1] - 20),
(bbox[0] + bbox[2] + 20, bbox[1] + bbox[3] + 20),
(255, 0, 255), 2)
cv2.putText(img, myHand["type"]
, (bbox[0] - 30, bbox[1] - 30)
, cv2.FONT_HERSHEY_PLAIN
, 2, (255, 0, 255), 2)
self.mpDraw.draw_landmarks(img, handLms
, self.mpHands.HAND_CONNECTIONS
, self.mpDraw.DrawingSpec(color=(0, 0, 255)
, thickness=t
, circle_radius=r)
, self.mpDraw.DrawingSpec(color=(0, 255, 0)
, thickness=2))
if draw:
return allHands, img
else:
return allHands
def fingersUp(self, myHand):
"""
Finds how many fingers are open and returns in a list.
Considers left and right hands separately
:return: List of which fingers are up
"""
myHandType =myHand["type"]
myLmList = myHand["lmList"]
if self.results.multi_hand_landmarks:
fingers = []
# Thumb
if myHandType == "Right":
if myLmList[self.tipIds[0]-1][1] > myLmList[self.tipIds[0]][1]:
fingers.append(1)
else:
fingers.append(0)
else:
if myLmList[self.tipIds[0]][1] > myLmList[self.tipIds[0]-1][1]:
fingers.append(1)
else:
fingers.append(0)
# 4 Fingers
for id in range(1, 5):
if myLmList[self.tipIds[id]][2] < myLmList[self.tipIds[id] - 2][2]:
fingers.append(1)
else:
fingers.append(0)
return fingers
def findDistance(self, p1, p2, myHand, img=None, draw=True, r=15, t=3):
"""
Find the distance between two landmarks based on their
index numbers.
:param p1: Point1,int:0-20
:param p2: Point2,int:0-20
:param img: Image to draw on.
:param draw: Flag to draw the output on the image.
:return: Distance between the points
Image with output drawn
Line information
"""
if len(myHand) == 1:
myHandType = myHand[0]["type"]
myLmList = myHand[0]["lmList"]
x1, y1 = myLmList[p1][1:]
x2, y2 = myLmList[p2][1:]
elif len(myHand) == 2:
myHandType1 = myHand[0]["type"]
myLmList1 = myHand[0]["lmList"]
myHandType2 = myHand[1]["type"]
myLmList2 = myHand[1]["lmList"]
x1, y1 = myLmList1[p1][1:]
x2, y2 = myLmList2[p2][1:]
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
# math.hypot()函数，返回所有参数的平方和的平方根
length = math.hypot(x2-x1, y2-y1)
lineInfo = (x1, y1, x2, y2, cx, cy)
if img is not None:
if draw:
cv2.line(img, (x1, y1), (x2, y2), (0, 255, 0), thickness=t)
cv2.circle(img, (x1, y1), r, (255, 0, 255), cv2.FILLED)
cv2.circle(img, (x2, y2), r, (255, 0, 255), cv2.FILLED)
cv2.circle(img, (cx, cy), r, (255, 0, 255), cv2.FILLED)
return length, lineInfo, img
else:
return length, info
def main():
##########
wCamera = 1280
hCamera = 720
##########
cap = cv2.VideoCapture(0)
cap.set(3, wCamera)
cap.set(4, hCamera)
detector = HandDetector(detectionCon=0.8)
while True:
# Get image frame (BGR存储格式)
success, img = cap.read()
img = cv2.flip(img, flipCode=1)
# Find the hand and its landmarks
# the type of hands are dictionary, likes[{"lmList","bbox","center","type"},...]
hands, img = detector.findHands(img, flipType=False) # with draw
# hands = detector.findHands(img, draw=False) # without draw
if hands:
# Hand 1
hand1 = hands[0]
lmList1 = hand1["lmList"] # List of 21 Landmark points
bbox1 = hand1["bbox"] # Bounding box info x,y,w,h
centerPoint1 = hand1['center'] # center of the hand cx,cy
handType1 = hand1["type"] # Handtype Left or Right
fingers1 = detector.fingersUp(hand1)
print("fingers1:", fingers1)
if len(hands) == 2:
# Hand 2
hand2 = hands[1]
lmList2 = hand2["lmList"] # List of 21 Landmark points
bbox2 = hand2["bbox"] # Bounding box info x,y,w,h
centerPoint2 = hand2['center'] # center of the hand cx,cy
handType2 = hand2["type"] # Hand Type "Left" or "Right"
fingers2 = detector.fingersUp(hand2)
print("fingers2:", fingers2)
# Find Distance between two Landmarks. Could be same hand or different hands
_, _, img = detector.findDistance(8, 12, [hand1], img, draw=True)
_, _, img = detector.findDistance(8, 8, [hand1, hand2]
, img, draw=True, r=15, t=3) # with draw
# length, lineinfo, img = detector.findDistance(8, 8, [hand1, hand2]) # no draw
cv2.imshow("Image", img)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
cv2.destroyAllWindows()
cap.release()
if __name__ == "__main__":
main()2.整体代码
代码如下（示例）：
import os
import cv2
import time
import math
import numpy as np
import HandTrackingModule as HTM
##########
wCamera, hCamera = 1280, 720
wImg, hImg = 1280, 150
beginXRect, beginYRect = 0, 160
endXRect, endYRect = 67, 700
WHITE_COLOR = (255, 255, 255)
RED_COLOR = (0, 0, 255)
YELLOW_COLOR = (0, 255, 255)
BLACK_COLOR = (0, 0, 0)
PURPLE_COLOR = (255, 0, 255)
BLUE_COLOR = (255, 0, 0)
brushThickness = 5
minBrushThickness = 0
maxBrushThickness = 15
eraserThickness = 10
wCanvas, hCanvas = 1280, 720
imgNum = 0
##########
global index, specFunc, drawColor
cap = cv2.VideoCapture(0)
cap.set(3, wCamera)
cap.set(4, hCamera)
# create canvas
canvas = np.zeros((hCanvas, wCanvas, 3), dtype=np.uint8)
canvas = 255 - canvas
xp, yp = 0, 0 # 画笔的起点
# check is or not click
class checkClick():
def __init__(self, img, lineInfo, overlayList):
self.img = img
self.lineInfo = lineInfo
self.cx, self.cy = self.lineInfo[-2], self.lineInfo[-1]
self.overlayList = overlayList
def clickBrush(self):
index = 0
board = self.overlayList[0]
drawColor = WHITE_COLOR
if 300 < self.cx < 400: # red
board = self.overlayList[1]
drawColor = RED_COLOR
elif 450 < self.cx < 550: # yellow
board = self.overlayList[2]
drawColor = YELLOW_COLOR
elif 590 < self.cx < 700: # black
board = self.overlayList[3]
drawColor = BLACK_COLOR
elif 750 < self.cx < 850: # blue
board = self.overlayList[4]
drawColor = BLUE_COLOR
elif 880 < self.cx < 1000: # pink
board = self.overlayList[5]
drawColor = PURPLE_COLOR
elif 1080 < self.cx < 1250: # eraser likes the color of white
board = self.overlayList[6]
drawColor = WHITE_COLOR
return index, board, drawColor
def clickFunc(self):
global specFunc, board
index = 1
board = self.overlayList[0]
if 200 < self.cy < 300: # circle
specFunc = "circle"
if 300 < self.cy < 400: # rectangle
specFunc = "rectangle"
if 400 < self.cy < 500: # triangle
specFunc = "triangle"
if 500 < self.cy < 700:
specFunc = "controlBrushThickness"
return index, board, specFunc
# 判断是否要画特殊图形，index=0：任意画；先选取颜色即令index=0，然后选形状即令index=1，从而绘制特殊形状
pointList = []
class drawSpecialGraph():
def __init__(self, img, canvas, center, color=(255, 0, 0), thickness=brushThickness):
self.img = img
self.canvas = canvas
self.center = center
self.color = color
self.thickness= thickness
self.center_x = self.center[0]
self.center_y = self.center[1]
def circle(self, thumb):
thumb_x = thumb[0]
thumb_y = thumb[1]
radius = int(math.hypot(self.center_x - thumb_x, self.center_y - thumb_y))
cv2.circle(self.img, (self.center_x, self.center_y), radius, self.color, self.thickness)
cv2.circle(self.canvas, (self.center_x, self.center_y), radius, self.color, self.thickness)
# def triangle(self, thumb):
# thumb_x = thumb[0]
# thumb_y = thumb[1]
# insideLength = int(math.hypot(self.center_x - thumb_x, self.center[1] - thumb_y))
# point1 = (self.center_x, self.center_y - insideLength)
# point2 = (self.center_x - 0.86 * insideLength, self.center_y + 0.5 * insideLength)
# point3 = (self.center_x + 0.86 * insideLength, self.center_y + 0.5 * insideLength)
# pointList = [point1, point2, point3]
# for i in range(3):
# cv2.line(self.img, (pointList[i][0], pointList[i][1])
# , (pointList[i + 1][0], pointList[i + 1][1]), self.color, self.thickness)
# cv2.line(self.canvas, (pointList[i][0], pointList[i][1])
# , (pointList[i + 1][0], pointList[i + 1][1]), self.color, self.thickness)
#
# def rectangle(self, thumb):
# thumb_x = thumb[0]
# thumb_y = thumb[1]
# insideLength = math.hypot(self.center_x - thumb_x, self.center[1] - thumb_y)
# point1 = (self.center_x - 0.86 * insideLength, self.center_y - 0.5 * insideLength)
# point2 = (self.center_x - 0.86 * insideLength, self.center_y + 0.5 * insideLength)
# point3 = (self.center_x + 0.86 * insideLength, self.center_y + 0.5 * insideLength)
# point4 = (self.center_x + 0.86 * insideLength, self.center_y - 0.5 * insideLength)
# pointList = [point1, point2, point3, point4]
# for i in range(4):
# cv2.line(self.img, (pointList[i][0], pointList[i][1])
# , (pointList[i + 1][0], pointList[i + 1][1]), self.color, self.thickness)
# cv2.line(self.canvas, (pointList[i][0], pointList[i][1])
# , (pointList[i + 1][0], pointList[i + 1][1]), self.color, self.thickness)
class specFunction():
def __init__(self, img, canvas, xp, yp):
self.img = img
self.canvas = canvas
self.xp, self.yp = xp, yp
def drawSpecShape(self, xp, yp, specFunc):
drawGraph = drawSpecialGraph(self.img, self.canvas, [self.xp, self.yp])
if specFunc == "circle":
time.sleep(1)
drawGraph.circle([thumb_x, thumb_y])
if specFunc == "triangle":
time.sleep(1)
drawGraph.triangle([thumb_x, thumb_y])
if specFunc == "rectangle":
time.sleep(1)
drawGraph.rectangle([thumb_x, thumb_y])
folderPath = "G:/2virtual_env python-learning-items/mediapip_cvzone/AI Virtual Painter/img"
savePath = "G:/2virtual_env python-learning-items/mediapip_cvzone/AI Virtual Painter/drawImg"
myList = os.listdir(folderPath)
# print(myList)
overlayList = []
for imgPath in myList:
img = cv2.imread(f"{folderPath}/{imgPath}")
overlayList.append(img)
# print(len(overlayList))
board = overlayList[0]
detector = HTM.HandDetector(maxHands=1, detectionCon=0.8)
while True:
## 1.Get image frame (BGR存储格式)
success, img = cap.read()
img = cv2.flip(img, 1)
## 2.Find hand landmarks
hands, img = detector.findHands(img, flipType=False, draw=True, drawRect=False) # with draw
## 3.Check which fingers are up
if hands:
hand1 = hands[0]
lmList1 = hand1["lmList"] # List of 21 Landmark points
bbox1 = hand1["bbox"] # Bounding box info x,y,w,h
centerPoint1 = hand1['center'] # center of the hand cx,cy
handType1 = hand1["type"] # Handtype Left or Right
fingers1 = detector.fingersUp(hand1)
# print("fingers1:", fingers1)
## 4.selection mode -- two finger are up
if fingers1 == [0, 1, 1, 0, 0]:
length, lineInfo, img = detector.findDistance(8, 12, [hand1], img, draw=True)
cx, cy = lineInfo[4], lineInfo[5] # the middle point between index and middle
# check for the click
if length < 60:
cv2.circle(img, (cx, cy), 15, (0, 255, 0), cv2.FILLED)
click = checkClick(img, lineInfo, overlayList)
if cy < 127:
index, board, drawColor = click.clickBrush()
if cx < 67:
index, board, specFunc = click.clickFunc()
## 5.drawing mode -- index finger is up
if fingers1 == [0, 1, 0, 0, 0]:
thumb_x, thumb_y = lmList1[8][1], lmList1[8][2]
cv2.circle(img, (thumb_x, thumb_y), 15, drawColor, cv2.FILLED)
# 5.1 click the brush region
if index == 0:
if xp == 0 and yp == 0:
xp, yp = thumb_x, thumb_y
if drawColor == (255, 255, 255):
cv2.line(img, (xp, yp), (thumb_x, thumb_y), drawColor, eraserThickness)
cv2.line(canvas, (xp, yp), (thumb_x, thumb_y), drawColor, eraserThickness)
else:
cv2.line(img, (xp, yp), (thumb_x, thumb_y), drawColor, brushThickness)
cv2.line(canvas, (xp, yp), (thumb_x, thumb_y), drawColor, brushThickness)
xp, yp = thumb_x, thumb_y # update the position of brush
# 5.2 click the specFunc region
if index == 1:
if xp == 0 and yp == 0:
xp, yp = thumb_x, thumb_y # define the center of special shape
SFunc = specFunction(img, canvas, xp, yp)
if specFunc == "controlBrushThickness":
brushThickness += 1
if brushThickness >= maxBrushThickness:
brushThickness = minBrushThickness
if brushThickness <= minBrushThickness:
brushThickness = maxBrushThickness
else:
SFunc.drawSpecShape(xp, yp, specFunc)
xp, yp = thumb_x, thumb_y # update the position of brush
##6 show the canvas and img
img[0:hImg, 0:wImg] = board[0:hImg, 0:wImg]
img[beginYRect - 10:endYRect - 10, beginXRect:endXRect] = board[beginYRect:endYRect, beginXRect:endXRect]
img = cv2.addWeighted(img, 0.8, canvas, 0.2, 0)
cv2.imshow("PaintBoard", img)
cv2.putText(canvas, "canvas", (wCanvas // 3, 80), cv2.FONT_HERSHEY_PLAIN, 7, (255, 255, 0), 7)
cv2.imshow("Canvas", canvas)
if cv2.waitKey(1) & 0xFF == ord("s"): # 按s键保存canvas
imgPath = str(imgNum) + ".jpg"
cv2.imwrite(f"{savePath}/{imgPath}", canvas)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
cv2.destroyAllWindows()
cap.release()
三、效果如下
由于程序的设计，在画圆之前和之后都要点画笔；增加线宽后，再绘画也必须重新点画笔。
总结:
通过对手部21个关节点进行跟踪识别，得到的标记点的坐标从而来进行相关的判断和控制。主要实现的功能包括绘图（任意画线、画圆（不太稳定）、画矩形和画三角形（暂时还没有实现））、调节线条的宽度、保存。