基于CNN+Opencv人脸识别【判断情绪】

序

判断情绪效果

CNN分类训练原理

训练

测试

多角度-多层次训练

人脸数据集

提取特征点

提取特征点

CNN分类

预测模型

对比分类

结果

正文

环境：Windows10 x64、python 3.7、anaconda3、keras 2.2.4、OpenCV3.4.1

一、利用机器学习模型训练和检测笑脸

Training and testing smile-detector with machine learning models

这部分为参考学习，转载原处：
Author: coneypo
Blog: http://www.cnblogs.com/AdaminXie/
Github: https://github.com/coneypo/Smile_Detector （这里可以下载数据集）

数据集分类目录：

（1） get_features.py

• 输入人脸图像路径;

• 利用 Dlib 的 “shape_predictor_68_face_landmarks.dat” 提取嘴部20个特征点坐标的40个特征值;

• write_into_CSV() 将 40 维特征输入和1维的输出标记写入 data.csv;
  

import dlib         # 人脸处理的库 Dlib
import numpy as np  # 数据处理的库 numpy
import cv2          # 图像处理的库 OpenCv
import os           # 读取文件
import csv          # CSV 操作


detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('data/data_dlib_model/shape_predictor_68_face_landmarks.dat')


# 输入图像文件所在路径，返回一个41维数组（包含提取到的40维特征和1维输出标记）
def get_features(img_rd):

    # 输入:  img_rd:      图像文件
    # 输出:  positions_lip_arr:  feature point 49 to feature point 68, 20 feature points / 40D in all

    # read img file
    img = cv2.imread(img_rd)
    # 取灰度
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # 计算68点坐标
    positions_68_arr = []
    faces = detector(img_gray, 0)
    landmarks = np.matrix([[p.x, p.y] for p in predictor(img, faces[0]).parts()])

    for idx, point in enumerate(landmarks):
        # 68点的坐标
        pos = (point[0, 0], point[0, 1])
        positions_68_arr.append(pos)

    positions_lip_arr = []
    # 将点 49-68 写入 CSV
    # 即 positions_68_arr[48]-positions_68_arr[67]
    for i in range(48, 68):
        positions_lip_arr.append(positions_68_arr[i][0])
        positions_lip_arr.append(positions_68_arr[i][1])

    return positions_lip_arr


# 读取图像所在的路径
path_images_with_smiles = "data/data_imgs/database/smiles/"
path_images_no_smiles = "data/data_imgs/database/no_smiles/"

# 获取路径下的图像文件
imgs_smiles = os.listdir(path_images_with_smiles)
imgs_no_smiles = os.listdir(path_images_no_smiles)

# 存储提取特征数据的 CSV 的路径
path_csv = "data/data_csvs/"


# write the features into CSV
def write_into_CSV():
    with open(path_csv+"data.csv", "w", newline="") as csvfile:
        writer = csv.writer(csvfile)

        # 处理带笑脸的图像
        print("######## with smiles #########")
        for i in range(len(imgs_smiles)):
            print(path_images_with_smiles+imgs_smiles[i])

            # append "1" means "with smiles"
            features_csv_smiles = get_features(path_images_with_smiles+imgs_smiles[i])
            features_csv_smiles.append(1)
            print("positions of lips:", features_csv_smiles, "\n")

            # 写入CSV
            writer.writerow(features_csv_smiles)

        # 处理不带笑脸的图像
        print("######## no smiles #########")
        for i in range(len(imgs_no_smiles)):
            print(path_images_no_smiles+imgs_no_smiles[i])

            # append "0" means "no smiles"
            features_csv_no_smiles = get_features(path_images_no_smiles + imgs_no_smiles[i])
            features_csv_no_smiles.append(0)
            print("positions of lips:", features_csv_no_smiles, "\n")

            # 写入CSV
            writer.writerow(features_csv_no_smiles)


# 写入CSV
# write_into_CSV()

（2） ML_ways_sklearn.py（线性回归模型、多层神经网络模型、线性SVM模型、梯度下降模型：）

• 读取 data.csv 中的数据, 然后提取出训练集 X_train 和测试集 X_test;
• train and test by sklearn

# pandas 读取 CSV
import pandas as pd

# 分割数据
from sklearn.model_selection import train_test_split

# 用于数据预加工标准化
from sklearn.preprocessing import StandardScaler

from sklearn.linear_model import LogisticRegression     # 线性模型中的 Logistic 回归模型
from sklearn.neural_network import MLPClassifier        # 神经网络模型中的多层网络模型
from sklearn.svm import LinearSVC                       # SVM 模型中的线性 SVC 模型
from sklearn.linear_model import SGDClassifier          # 线性模型中的随机梯度下降模型

from sklearn.externals import joblib


# 从 csv 读取数据
def pre_data():
    # 41 维表头
    column_names = []
    for i in range(0, 40):
        column_names.append("feature_" + str(i + 1))
    column_names.append("output")

    # read csv
    rd_csv = pd.read_csv("data/data_csvs/data.csv", names=column_names)

    # 输出 csv 文件的维度
    # print("shape:", rd_csv.shape)

    X_train, X_test, y_train, y_test = train_test_split(

        # input 0-40
        # output 41
        rd_csv[column_names[0:40]],
        rd_csv[column_names[40]],

        # 25% for testing, 75% for training
        test_size=0.25,
        random_state=33)

    return X_train, X_test, y_train, y_test


path_models = "data/data_models/"


# LR, logistic regression, 逻辑斯特回归分类（线性模型）
def model_LR():
    # get data
    X_train_LR, X_test_LR, y_train_LR, y_test_LR = pre_data()

    # 数据预加工
    # 标准化数据，保证每个维度的特征数据方差为1，均值为0。使得预测结果不会被某些维度过大的特征值而主导
    ss_LR = StandardScaler()
    X_train_LR = ss_LR.fit_transform(X_train_LR)
    X_test_LR = ss_LR.transform(X_test_LR)

    # 初始化 LogisticRegression
    LR = LogisticRegression()

    # 调用 LogisticRegression 中的 fit() 来训练模型参数
    LR.fit(X_train_LR, y_train_LR)

    # save LR model
    joblib.dump(LR, path_models + "model_LR.m")

    # 评分函数
    score_LR = LR.score(X_test_LR, y_test_LR)
    print("The accurary of LR:", score_LR)

    # print(type(ss_LR))
    return (ss_LR)


# model_LR()


# MLPC, Multi-layer Perceptron Classifier, 多层感知机分类（神经网络）
def model_MLPC():
    # get data
    X_train_MLPC, X_test_MLPC, y_train_MLPC, y_test_MLPC = pre_data()

    # 数据预加工
    ss_MLPC = StandardScaler()
    X_train_MLPC = ss_MLPC.fit_transform(X_train_MLPC)
    X_test_MLPC = ss_MLPC.transform(X_test_MLPC)

    # 初始化 MLPC
    MLPC = MLPClassifier(hidden_layer_sizes=(13, 13, 13), max_iter=500)

    # 调用 MLPC 中的 fit() 来训练模型参数
    MLPC.fit(X_train_MLPC, y_train_MLPC)

    # save MLPC model
    joblib.dump(MLPC, path_models + "model_MLPC.m")

    # 评分函数
    score_MLPC = MLPC.score(X_test_MLPC, y_test_MLPC)
    print("The accurary of MLPC:", score_MLPC)

    return (ss_MLPC)


# model_MLPC()


# Linear SVC， Linear Supported Vector Classifier, 线性支持向量分类(SVM支持向量机)
def model_LSVC():
    # get data
    X_train_LSVC, X_test_LSVC, y_train_LSVC, y_test_LSVC = pre_data()

    # 数据预加工
    ss_LSVC = StandardScaler()
    X_train_LSVC = ss_LSVC.fit_transform(X_train_LSVC)
    X_test_LSVC = ss_LSVC.transform(X_test_LSVC)

    # 初始化 LSVC
    LSVC = LinearSVC()

    # 调用 LSVC 中的 fit() 来训练模型参数
    LSVC.fit(X_train_LSVC, y_train_LSVC)

    # save LSVC model
    joblib.dump(LSVC, path_models + "model_LSVC.m")

    # 评分函数
    score_LSVC = LSVC.score(X_test_LSVC, y_test_LSVC)
    print("The accurary of LSVC:", score_LSVC)

    return ss_LSVC


# model_LSVC()


# SGDC, Stochastic Gradient Decent Classifier, 随机梯度下降法求解(线性模型)
def model_SGDC():
    # get data
    X_train_SGDC, X_test_SGDC, y_train_SGDC, y_test_SGDC = pre_data()

    # 数据预加工
    ss_SGDC = StandardScaler()
    X_train_SGDC = ss_SGDC.fit_transform(X_train_SGDC)
    X_test_SGDC = ss_SGDC.transform(X_test_SGDC)

    # 初始化 SGDC
    SGDC = SGDClassifier(max_iter=5)

    # 调用 SGDC 中的 fit() 来训练模型参数
    SGDC.fit(X_train_SGDC, y_train_SGDC)

    # save SGDC model
    joblib.dump(SGDC, path_models + "model_SGDC.m")

    # 评分函数
    score_SGDC = SGDC.score(X_test_SGDC, y_test_SGDC)
    print("The accurary of SGDC:", score_SGDC)

    return ss_SGDC

# model_SGDC()

（3） check_smile.py

• 利用模型测试图像文件中人脸是否微笑

# use the saved model
from sklearn.externals import joblib

from get_features import get_features
import ML_ways_sklearn

import cv2

# path of test img
path_test_img = "data/data_imgs/test_imgs/test1.jpg"

# 提取单张40维度特征
positions_lip_test = get_features(path_test_img)

# path of models
path_models = "data/data_models/"

print("The result of"+path_test_img+":")
print('\n')

# #########  LR  ###########
LR = joblib.load(path_models+"model_LR.m")
ss_LR = ML_ways_sklearn.model_LR()
X_test_LR = ss_LR.transform([positions_lip_test])
y_predict_LR = str(LR.predict(X_test_LR)[0]).replace('0', "no smile").replace('1', "with smile")
print("LR:", y_predict_LR)

# #########  LSVC  ###########
LSVC = joblib.load(path_models+"model_LSVC.m")
ss_LSVC = ML_ways_sklearn.model_LSVC()
X_test_LSVC = ss_LSVC.transform([positions_lip_test])
y_predict_LSVC = str(LSVC.predict(X_test_LSVC)[0]).replace('0', "no smile").replace('1', "with smile")
print("LSVC:", y_predict_LSVC)

# #########  MLPC  ###########
MLPC = joblib.load(path_models+"model_MLPC.m")
ss_MLPC = ML_ways_sklearn.model_MLPC()
X_test_MLPC = ss_MLPC.transform([positions_lip_test])
y_predict_MLPC = str(MLPC.predict(X_test_MLPC)[0]).replace('0', "no smile").replace('1', "with smile")
print("MLPC:", y_predict_MLPC)

# #########  SGDC  ###########
SGDC = joblib.load(path_models+"model_SGDC.m")
ss_SGDC = ML_ways_sklearn.model_SGDC()
X_test_SGDC = ss_SGDC.transform([positions_lip_test])
y_predict_SGDC = str(SGDC.predict(X_test_SGDC)[0]).replace('0', "no smile").replace('1', "with smile")
print("SGDC:", y_predict_SGDC)

img_test = cv2.imread(path_test_img)

img_height = int(img_test.shape[0])
img_width = int(img_test.shape[1])

# show the results on the image
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img_test, "LR:    "+y_predict_LR,   (int(img_height/10), int(img_width/10)), font, 0.8, (84, 255, 159), 1, cv2.LINE_AA)
cv2.putText(img_test, "LSVC:  "+y_predict_LSVC, (int(img_height/10), int(img_width/10*2)), font, 0.8, (84, 255, 159), 1, cv2.LINE_AA)
cv2.putText(img_test, "MLPC:  "+y_predict_MLPC, (int(img_height/10), int(img_width/10)*3), font, 0.8, (84, 255, 159), 1, cv2.LINE_AA)
cv2.putText(img_test, "SGDC:  "+y_predict_SGDC, (int(img_height/10), int(img_width/10)*4), font, 0.8, (84, 255, 159), 1, cv2.LINE_AA)

cv2.namedWindow("img", 2)
cv2.imshow("img", img_test)
cv2.waitKey(0)

（4）show_lip.py

• 显示某人嘴唇的位置

# 显示嘴部特征点
# Draw the positions of someone's lip

import dlib         # 人脸识别的库 Dlib
import cv2          # 图像处理的库 OpenCv
from get_features import get_features   # return the positions of feature points

path_test_img = "data/data_imgs/test_imgs/test1.jpg"

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('data/data_dlib_model/shape_predictor_68_face_landmarks.dat')

# Get lip's positions of features points
positions_lip = get_features(path_test_img)

img_rd = cv2.imread(path_test_img)

# Draw on the lip points
for i in range(0, len(positions_lip), 2):
    print(positions_lip[i], positions_lip[i+1])
    cv2.circle(img_rd, tuple([positions_lip[i], positions_lip[i+1]]), radius=1, color=(0, 255, 0))

cv2.namedWindow("img_read", 2)
cv2.imshow("img_read", img_rd)
cv2.waitKey(0)

（5）视频检测——check_smile_from_camera.py

# use the saved model
from sklearn.externals import joblib

import ML_ways_sklearn

import dlib         # 人脸处理的库 Dlib
import numpy as np  # 数据处理的库 numpy
import cv2          # 图像处理的库 OpenCv


detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('data/data_dlib_model/shape_predictor_68_face_landmarks.dat')

# OpenCv 调用摄像头
cap = cv2.VideoCapture(0)

# 设置视频参数
cap.set(3, 480)


def get_features(img_rd):

    # 输入:  img_rd:      图像文件
    # 输出:  positions_lip_arr:  feature point 49 to feature point 68, 20 feature points / 40D in all

    # 取灰度
    img_gray = cv2.cvtColor(img_rd, cv2.COLOR_RGB2GRAY)

    # 计算68点坐标
    positions_68_arr = []
    faces = detector(img_gray, 0)
    landmarks = np.matrix([[p.x, p.y] for p in predictor(img_rd, faces[0]).parts()])

    for idx, point in enumerate(landmarks):
        # 68点的坐标
        pos = (point[0, 0], point[0, 1])
        positions_68_arr.append(pos)

    positions_lip_arr = []
    # 将点 49-68 写入 CSV
    # 即 positions_68_arr[48]-positions_68_arr[67]
    for i in range(48, 68):
        positions_lip_arr.append(positions_68_arr[i][0])
        positions_lip_arr.append(positions_68_arr[i][1])

    return positions_lip_arr


while cap.isOpened():
    # 480 height * 640 width
    flag, img_rd = cap.read()
    kk = cv2.waitKey(1)

    img_gray = cv2.cvtColor(img_rd, cv2.COLOR_RGB2GRAY)

    # 人脸数 faces
    faces = detector(img_gray, 0)
    # 检测到人脸
    if len(faces) != 0:
        # 提取单张40维度特征
        positions_lip_test = get_features(img_rd)

        # path of models
        path_models = "data/data_models/"

        # #########  LR  ###########
        LR = joblib.load(path_models+"model_LR.m")
        ss_LR = ML_ways_sklearn.model_LR()
        X_test_LR = ss_LR.transform([positions_lip_test])
        y_predict_LR = str(LR.predict(X_test_LR)[0]).replace('0', "no smile").replace('1', "with smile")
        print("LR:", y_predict_LR)

        # #########  LSVC  ###########
        LSVC = joblib.load(path_models+"model_LSVC.m")
        ss_LSVC = ML_ways_sklearn.model_LSVC()
        X_test_LSVC = ss_LSVC.transform([positions_lip_test])
        y_predict_LSVC = str(LSVC.predict(X_test_LSVC)[0]).replace('0', "no smile").replace('1', "with smile")
        print("LSVC:", y_predict_LSVC)

        # #########  MLPC  ###########
        MLPC = joblib.load(path_models+"model_MLPC.m")
        ss_MLPC = ML_ways_sklearn.model_MLPC()
        X_test_MLPC = ss_MLPC.transform([positions_lip_test])
        y_predict_MLPC = str(MLPC.predict(X_test_MLPC)[0]).replace('0', "no smile").replace('1', "with smile")
        print("MLPC:", y_predict_MLPC)

        # #########  SGDC  ###########
        SGDC = joblib.load(path_models+"model_SGDC.m")
        ss_SGDC = ML_ways_sklearn.model_SGDC()
        X_test_SGDC = ss_SGDC.transform([positions_lip_test])
        y_predict_SGDC = str(SGDC.predict(X_test_SGDC)[0]).replace('0', "no smile").replace('1', "with smile")
        print("SGDC:", y_predict_SGDC)

        print('\n')

        # 按下 'q' 键退出
        if kk == ord('q'):
            break

    # 窗口显示
    # cv2.namedWindow("camera", 0) # 如果需要摄像头窗口大小可调
    cv2.imshow("camera", img_rd)

# 释放摄像头
cap.release()

# 删除建立的窗口
cv2.destroyAllWindows()

二、 扩展

从视频检测中，可以看出训练效果不是很好

解决方法：因为从前面学习中我们可以发现，样本数据集过少，训练模型较差；这里给出一个简单的基于CNN训练好的模型（对于入门者来说，也是找了好久），这里贴出来供大家学习。

数据集：./simple_CNN.530-0.65.hdf5——》 表情识别模板下载链接

#coding=utf-8
#表情识别

import cv2
from keras.models import load_model
import numpy as np
import datetime

startTime = datetime.datetime.now()
emotion_classifier = load_model(
    './simple_CNN.530-0.65.hdf5')
endTime = datetime.datetime.now()
print(endTime - startTime)

emotion_labels = {
    0: 'angry',
    1: 'disgust',
    2: 'fear',
    3: 'happy',
    4: 'sad',
    5: 'surprise',
    6: 'calm'
}

img = cv2.imread("./emotion.png")
face_classifier = cv2.CascadeClassifier(
    "D:/mydownload/opencv341/opencv/build/etc/haarcascades/haarcascade_frontalface_default.xml"
)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_classifier.detectMultiScale(
    gray, scaleFactor=1.2, minNeighbors=3, minSize=(40, 40))
color = (255, 0, 0)
font = cv2.FONT_HERSHEY_SIMPLEX  # 定义字体
for (x, y, w, h) in faces: # 矩形框位置
    gray_face = gray[(y):(y + h), (x):(x + w)]
    gray_face = cv2.resize(gray_face, (48, 48))
    gray_face = gray_face / 255.0
    gray_face = np.expand_dims(gray_face, 0)
    gray_face = np.expand_dims(gray_face, -1)
    emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
    emotion = emotion_labels[emotion_label_arg]
    print(emotion)
    cv2.rectangle(img, (x + 10, y + 10), (x + h - 10, y + w - 10),
                  color, 2)
    img = img.copy()  # 备份操作
    img2 = cv2.putText(img, emotion, (int(x + h * 0.3), int(y)), font, 1, color, 2)
                  # 图像，文字内容， 坐标 ，字体，大小，颜色，字体厚度

cv2.imshow("Image", img2)
cv2.waitKey(0)
cv2.destroyAllWindows()

---

附：
1、更多人脸识别示例参考：https://github.com/cungudafa/cungudafa.github.io