opencv dlib 总
test.py
import cv2 as cv
import dlib
import imutils
import numpy as np
from imutils import face_utils as fu
# 计算眼睛的纵横比
def eye_aspect_ratio(eye):
# 计算上下的欧式距离
a = np.linalg.norm(eye[1] - eye[5])
b = np.linalg.norm(eye[2] - eye[4])
# 计算左右的欧式距离
c = np.linalg.norm(eye[0] - eye[3])
# 计算纵横比并返回
d = (a + b) / (2.0 * c)
return d
# 计算嘴巴的纵横比
def mouth_aspect_ratio(mouth):
# 计算上下的欧式距离
a = np.linalg.norm(mouth[3] - mouth[9]) # 52, 59
b = np.linalg.norm(mouth[14] - mouth[18]) # 63, 67
# 计算左右的欧式距离
c = np.linalg.norm(mouth[0] - mouth[6]) # 49, 55
d = np.linalg.norm(mouth[12] - mouth[16]) # 61, 65
# 计算纵横比并返回
e = (a + b) / (c + d)
return e
# 连续3帧眨眼次数
EYE_COUNTER = 0
# 总共眨眼次数
EYE_TOTAL = 0
# 连续3帧张嘴次数
MOUTH_COUNTER = 0
# 总共张嘴次数
MOUTH_TOTAL = 0
# 打开摄像头
cap = cv.VideoCapture(0, cv.CAP_DSHOW)
# 建立人脸检测器
detector = dlib.get_frontal_face_detector()
# 建立68关键点检测器
predictor = dlib.shape_predictor('./1.dat')
# 返回人脸的左眼和右眼在68关键点中的起始和结束
(lStart, lEnd) = fu.FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = fu.FACIAL_LANDMARKS_IDXS["right_eye"]
# 返回人脸的嘴巴在68关键点中的起始和结束
(mStart, mEnd) = fu.FACIAL_LANDMARKS_IDXS["mouth"]
# 只要能正确打开摄像头
while cap.isOpened():
# 获取每一帧
_, frame = cap.read()
# 设置输出宽度
frame = imutils.resize(frame, width=750)
# 转变为灰度图加快识别
gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
# 返回这一帧的所有人脸框
faces = detector(gray, 0)
# 遍历所有框框
for face in faces:
# 返回该框的68个坐标
shape = predictor(gray, face)
# 转变为坐标矩阵
shape = fu.shape_to_np(shape)
# 返回左眼和右眼的坐标
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
# 计算左眼和右眼的纵横比
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
# 取平均值
earAVG = (leftEAR + rightEAR) / 2.0
# 计算左眼和右眼的凸包
leftEyeHull = cv.convexHull(leftEye)
rightEyeHull = cv.convexHull(rightEye)
# 圈出凸包，即眼睛的范围
cv.drawContours(frame, [leftEyeHull], -1, (255, 255, 255), 1)
cv.drawContours(frame, [rightEyeHull], -1, (255, 255, 255), 1)
# 返回嘴巴的坐标
Mouth = shape[mStart:mEnd]
# 计算嘴巴的纵横比
mar = mouth_aspect_ratio(Mouth)
# 计算嘴巴的凸包
MouthHull = cv.convexHull(Mouth)
# 圈出凸包，即嘴巴的范围
cv.drawContours(frame, [MouthHull], -1, (255, 255, 255), 1)
'''
若眼睛纵横比小于0.3 且 连续3帧
则认为闭眼
'''
if earAVG < 0.3:
EYE_COUNTER += 1
else:
if EYE_COUNTER >= 3:
EYE_TOTAL += 1
EYE_COUNTER = 0
'''
若嘴巴纵横比大于0.5 且 连续3帧
则认为打哈欠
'''
if mar > 0.5:
MOUTH_COUNTER += 1
else:
if MOUTH_COUNTER >= 3:
MOUTH_TOTAL += 1
MOUTH_COUNTER = 0
# 接下来是对输出的处理
# 左上角输出眨眼次数
cv.putText(
frame,
"Blinks:{0}".format(EYE_TOTAL),
(10, 20),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 255, 255),
2,
cv.LINE_AA
)
# 紧接实时earAVG
cv.putText(
frame,
"earAVG:{0}".format(earAVG),
(200, 20),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 255, 255),
2,
cv.LINE_AA
)
# 左上角输出打哈欠次数
cv.putText(
frame,
"Yawning:{0}".format(MOUTH_TOTAL),
(10, 50),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 255, 255),
2,
cv.LINE_AA
)
# 紧接实时mar
cv.putText(
frame,
"mar:{0}".format(mar),
(200, 50),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 255, 255),
2,
cv.LINE_AA
)
# 右下角提示退出信息
cv.putText(
frame,
"Press 'Esc' to Quit",
(515, 550),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 255, 255),
2,
cv.LINE_AA
)
# 输出每一帧
cv.imshow('camera', frame)
# 按Esc退出
if cv.waitKey(1) & 0xff == 27:
break
# 关闭所有窗口
cv.destroyAllWindows()
# 释放摄像头
cap.release()
main.py
import cv2 as cv
import dlib
import imutils
from imutils import face_utils as fu
import eye
import head
import mouth
# 连续3帧眨眼次数
EYE_COUNTER = 0
# 总共眨眼次数
EYE_TOTAL = 0
# 持续闭眼计数(单位为帧)
ALWAYS_CLOSE_EYES_COUNTER = 0
# 持续闭眼警告标志
AL_eye_flag = False
# 初始化帧计数器和点头总数
hCOUNTER = 0
hTOTAL = 0
# 持续低头计数(单位为帧)
ALWAYS_CLOSE_HEAD_COUNTER = 0
# 持续低头警告标志
AL_head_flag = False
# 连续3帧张嘴次数
MOUTH_COUNTER = 0
# 总共张嘴次数
MOUTH_TOTAL = 0
# 打开摄像头
cap = cv.VideoCapture(0, cv.CAP_DSHOW)
# 建立人脸检测器
detector = dlib.get_frontal_face_detector()
# 建立68关键点检测器
predictor = dlib.shape_predictor('./1.dat')
while cap.isOpened():
# 获取每一帧
_, frame = cap.read()
# 设置输出宽度
frame = imutils.resize(frame, width=750)
# 转变为灰度图加快识别
gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
# 返回这一帧的所有人脸框
faces = detector(gray, 0)
# 遍历所有框框
for face in faces:
# 返回该框的68个坐标
shape = predictor(gray, face)
# 转变为坐标矩阵
shape = fu.shape_to_np(shape)
# 计算左右眼纵横比的平均值
earAVG = eye.eye_detecting(frame, shape)
# 计算嘴巴的纵横比
mar = mouth.mouth_detecting(frame, shape)
# 获取头部姿态
reprojectdst, euler_angle = head.get_head_pose(shape)
'''
若眼睛纵横比小于0.3 且 连续3帧
则认为闭眼
'''
if earAVG < 0.3:
EYE_COUNTER += 1
ALWAYS_CLOSE_EYES_COUNTER += 1
else:
if EYE_COUNTER >= 3:
EYE_TOTAL += 1
EYE_COUNTER = 0
ALWAYS_CLOSE_EYES_COUNTER = 0
# 持续闭眼判断
if ALWAYS_CLOSE_EYES_COUNTER >= 18:
AL_eye_flag = True
else:
AL_eye_flag = False
'''
若嘴巴纵横比大于0.5 且 连续3帧
则认为打哈欠
'''
if mar > 0.5:
MOUTH_COUNTER += 1
else:
if MOUTH_COUNTER >= 3:
MOUTH_TOTAL += 1
MOUTH_COUNTER = 0
'''
头部pitch大于0.3 且 连续3帧
则认为点头一次
'''
har = euler_angle[0, 0] # 取pitch旋转角度
if har > 15: # 点头阈值15
hCOUNTER += 1
ALWAYS_CLOSE_HEAD_COUNTER += 1
else:
# 如果连续3次都小于阈值，则表示瞌睡点头一次
if hCOUNTER >= 3: # 阈值：3
hTOTAL += 1
# 重置点头帧计数器
hCOUNTER = 0
ALWAYS_CLOSE_HEAD_COUNTER = 0
if ALWAYS_CLOSE_HEAD_COUNTER >= 18:
AL_head_flag = True
else:
AL_head_flag = False
# 接下来是对输出的处理
# 左上角输出眨眼次数
cv.putText(
frame,
"Blinks:{0}".format(EYE_TOTAL),
(10, 20),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(0, 0, 255),
2,
cv.LINE_AA
)
# 紧接实时earAVG
cv.putText(
frame,
"earAVG:{0}".format(earAVG),
(200, 20),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(0, 0, 255),
2,
cv.LINE_AA
)
# 如果检测到超过2秒的持续闭眼则告警
if AL_eye_flag:
cv.putText(
frame,
"WARNING EYE",
(200, 150),
cv.FONT_HERSHEY_COMPLEX_SMALL,
2,
(100, 100, 255),
2,
cv.LINE_AA
)
# 左上角输出打哈欠次数
cv.putText(
frame,
"Yawning:{0}".format(MOUTH_TOTAL),
(10, 50),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(0, 0, 255),
2,
cv.LINE_AA
)
# 紧接实时mar
cv.putText(
frame,
"mar:{0}".format(mar),
(200, 50),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(0, 0, 255),
2,
cv.LINE_AA
)
for start, end in head.line_pairs:
a = reprojectdst[start]
b = reprojectdst[end]
x1 = int(a[0])
y1 = int(a[1])
x2 = int(b[0])
y2 = int(b[1])
cv.line(frame, (x1, y1), (x2, y2), (0, 0, 255), 1)
# 显示角度结果
cv.putText(
frame,
"X: " + "{:7.2f}".format(euler_angle[0, 0]),
(10, 90),
cv.FONT_HERSHEY_SIMPLEX,
0.75,
(0, 255, 0),
2
)
cv.putText(
frame,
"Y: " + "{:7.2f}".format(euler_angle[1, 0]),
(150, 90),
cv.FONT_HERSHEY_SIMPLEX,
0.75,
(255, 0, 0),
2
)
cv.putText(
frame,
"Z: " + "{:7.2f}".format(euler_angle[2, 0]),
(300, 90),
cv.FONT_HERSHEY_SIMPLEX,
0.75,
(0, 0, 255),
2
)
cv.putText(
frame,
"Nod: {}".format(hTOTAL),
(450, 90),
cv.FONT_HERSHEY_SIMPLEX,
0.7,
(255, 255, 0),
2
)
cv.putText(
frame,
"har: {}".format(har),
(10, 120),
cv.FONT_HERSHEY_SIMPLEX,
0.7,
(255, 255, 0),
2
)
# 如果检测到超过2秒的持续低头则告警
if AL_head_flag:
cv.putText(
frame,
"WARNING HEAD",
(200, 250),
cv.FONT_HERSHEY_COMPLEX_SMALL,
2,
(100, 100, 255),
2,
cv.LINE_AA
)
# 右下角提示退出信息
cv.putText(
frame,
"Press 'Esc' to Quit",
(515, 550),
cv.FONT_HERSHEY_COMPLEX_SMALL,
1,
(255, 0, 0),
2,
cv.LINE_AA
)
# 输出每一帧
cv.imshow('camera', frame)
# 按Esc退出
if cv.waitKey(1) & 0xff == 27:
break
# 关闭所有窗口
cv.destroyAllWindows()
# 释放摄像头
cap.release()
head.py
import math
import cv2 as cv
import numpy as np
# 世界坐标系(UVW)：填写3D参考点，该模型参考http://aifi.isr.uc.pt/Downloads/OpenGL/glAnthropometric3DModel.cpp
object_pts = np.float32([[6.825897, 6.760612, 4.402142], # 33左眉左上角
[1.330353, 7.122144, 6.903745], # 29左眉右角
[-1.330353, 7.122144, 6.903745], # 34右眉左角
[-6.825897, 6.760612, 4.402142], # 38右眉右上角
[5.311432, 5.485328, 3.987654], # 13左眼左上角
[1.789930, 5.393625, 4.413414], # 17左眼右上角
[-1.789930, 5.393625, 4.413414], # 25右眼左上角
[-5.311432, 5.485328, 3.987654], # 21右眼右上角
[2.005628, 1.409845, 6.165652], # 55鼻子左上角
[-2.005628, 1.409845, 6.165652], # 49鼻子右上角
[2.774015, -2.080775, 5.048531], # 43嘴左上角
[-2.774015, -2.080775, 5.048531], # 39嘴右上角
[0.000000, -3.116408, 6.097667], # 45嘴中央下角
[0.000000, -7.415691, 4.070434]]) # 6下巴角
# 相机坐标系(XYZ)：添加相机内参
K = [6.5308391993466671e+002, 0.0, 3.1950000000000000e+002,
0.0, 6.5308391993466671e+002, 2.3950000000000000e+002,
0.0, 0.0, 1.0] # 等价于矩阵[fx, 0, cx; 0, fy, cy; 0, 0, 1]
# 图像中心坐标系(uv)：相机畸变参数[k1, k2, p1, p2, k3]
D = [7.0834633684407095e-002, 6.9140193737175351e-002, 0.0, 0.0, -1.3073460323689292e+000]
# 像素坐标系(xy)：填写凸轮的本征和畸变系数
cam_matrix = np.array(K).reshape(3, 3).astype(np.float32)
dist_coeffs = np.array(D).reshape(5, 1).astype(np.float32)
# 重新投影3D点的世界坐标轴以验证结果姿势
reprojectsrc = np.float32([[10.0, 10.0, 10.0],
[10.0, 10.0, -10.0],
[10.0, -10.0, -10.0],
[10.0, -10.0, 10.0],
[-10.0, 10.0, 10.0],
[-10.0, 10.0, -10.0],
[-10.0, -10.0, -10.0],
[-10.0, -10.0, 10.0]])
# 绘制正方体12轴
line_pairs = [[0, 1], [1, 2], [2, 3], [3, 0],
[4, 5], [5, 6], [6, 7], [7, 4],
[0, 4], [1, 5], [2, 6], [3, 7]]
def get_head_pose(shape): # 头部姿态估计
# （像素坐标集合）填写2D参考点，注释遵循https://ibug.doc.ic.ac.uk/resources/300-W/
# 17左眉左上角/21左眉右角/22右眉左上角/26右眉右上角/36左眼左上角/39左眼右上角/42右眼左上角/
# 45右眼右上角/31鼻子左上角/35鼻子右上角/48左上角/54嘴右上角/57嘴中央下角/8下巴角
image_pts = np.float32([shape[17], shape[21], shape[22], shape[26], shape[36],
shape[39], shape[42], shape[45], shape[31], shape[35],
shape[48], shape[54], shape[57], shape[8]])
# solvePnP计算姿势——求解旋转和平移矩阵：
# rotation_vec表示旋转矩阵，translation_vec表示平移矩阵，cam_matrix与K矩阵对应，dist_coeffs与D矩阵对应。
_, rotation_vec, translation_vec = cv.solvePnP(object_pts, image_pts, cam_matrix, dist_coeffs)
# projectPoints重新投影误差：原2d点和重投影2d点的距离（输入3d点、相机内参、相机畸变、r、t，输出重投影2d点）
reprojectdst, _ = cv.projectPoints(reprojectsrc, rotation_vec, translation_vec, cam_matrix, dist_coeffs)
reprojectdst = tuple(map(tuple, reprojectdst.reshape(8, 2))) # 以8行2列显示
# print(type(reprojectdst))
# print("repro = {0}".format(reprojectdst))
# 计算欧拉角calc euler angle
# 参考https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#decomposeprojectionmatrix
rotation_mat, _ = cv.Rodrigues(rotation_vec) # 罗德里格斯公式（将旋转矩阵转换为旋转向量）
pose_mat = cv.hconcat((rotation_mat, translation_vec)) # 水平拼接，vconcat垂直拼接
# decomposeProjectionMatrix将投影矩阵分解为旋转矩阵和相机矩阵
_, _, _, _, _, _, euler_angle = cv.decomposeProjectionMatrix(pose_mat)
pitch, yaw, roll = [math.radians(_) for _ in euler_angle]
pitch = math.degrees(math.asin(math.sin(pitch)))
roll = -math.degrees(math.asin(math.sin(roll)))
yaw = math.degrees(math.asin(math.sin(yaw)))
# print('pitch:{}, yaw:{}, roll:{}'.format(pitch, yaw, roll))
return reprojectdst, euler_angle # 投影误差，欧拉角
eye.py
import cv2 as cv
import numpy as np
from imutils import face_utils as fu
# # 连续3帧眨眼次数
# EYE_COUNTER = 0
# # 总共眨眼次数
# EYE_TOTAL = 0
# 计算眼睛的纵横比
def eye_aspect_ratio(eye):
# 计算上下的欧式距离
a = np.linalg.norm(eye[1] - eye[5])
b = np.linalg.norm(eye[2] - eye[4])
# 计算左右的欧式距离
c = np.linalg.norm(eye[0] - eye[3])
# 计算纵横比并返回
d = (a + b) / (2.0 * c)
return d
# 返回左右眼纵横比的平均值
def eye_detecting(frame, shape):
# 返回人脸的左眼和右眼在68关键点中的起始和结束
(lStart, lEnd) = fu.FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = fu.FACIAL_LANDMARKS_IDXS["right_eye"]
# 返回左眼和右眼的坐标
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
# 计算左眼和右眼的纵横比
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
# 取平均值
earAVG = (leftEAR + rightEAR) / 2.0
# 计算左眼和右眼的凸包
leftEyeHull = cv.convexHull(leftEye)
rightEyeHull = cv.convexHull(rightEye)
# 圈出凸包，即眼睛的范围
cv.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
cv.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)
return earAVG
mouth.py
import cv2 as cv
import numpy as np
from imutils import face_utils as fu
# # 连续3帧张嘴次数
# MOUTH_COUNTER = 0
# # 总共张嘴次数
# MOUTH_TOTAL = 0
# 计算嘴巴的纵横比
def mouth_aspect_ratio(mouth):
# 计算上下的欧式距离
a = np.linalg.norm(mouth[3] - mouth[9]) # 52, 59
b = np.linalg.norm(mouth[14] - mouth[18]) # 63, 67
# 计算左右的欧式距离
c = np.linalg.norm(mouth[0] - mouth[6]) # 49, 55
d = np.linalg.norm(mouth[12] - mouth[16]) # 61, 65
# 计算纵横比并返回
e = (a + b) / (c + d)
return e
# 返回嘴巴的纵横比
def mouth_detecting(frame, shape):
# 返回人脸的嘴巴在68关键点中的起始和结束
(mStart, mEnd) = fu.FACIAL_LANDMARKS_IDXS["mouth"]
# 返回嘴巴的坐标
Mouth = shape[mStart:mEnd]
# 计算嘴巴的纵横比
mar = mouth_aspect_ratio(Mouth)
# 计算嘴巴的凸包
MouthHull = cv.convexHull(Mouth)
# 圈出凸包，即嘴巴的范围
cv.drawContours(frame, [MouthHull], -1, (0, 255, 0), 1)
return mar