根据人脸关键点做人脸对齐face alignment----C++实现

目录

1 人脸检测

 2 人脸对齐代码

2.1 FaceProprocess.h

2.2 main.cpp

2.3 makefile

3 对齐效果

---

人脸识别一般需要经过人脸检测，人脸对齐，特征提取及相似度比对，具体原理可以见：

人脸检测-人脸对齐-人脸识别原理及方法_陈 洪 伟的博客-CSDN博客

该篇文章主要对人脸对齐进行补充，

1 人脸检测

我们用ldh的图片进行检测，得到人脸框和5个关键点。

 检测得到的矩形框和关键点如下：

orgimg.shape:  (640, 640, 3)
x1:243.000000,y1:140.000000,x2:447.000000,y2:394.000000
point_0_x:307.000000
point_0_y:252.000000
point_1_x:390.000000
point_1_y:241.000000
point_2_x:359.000000
point_2_y:305.000000
point_3_x:332.000000
point_3_y:340.000000
point_4_x:395.000000
point_4_y:332.000000

结果图如下

 2 人脸对齐代码

2.1 FaceProprocess.h

//
// Created by Jack Yu on 23/03/2018.
//
 
#ifndef FACE_DEMO_FACEPREPROCESS_H
#define FACE_DEMO_FACEPREPROCESS_H
 
#include
#include 
 
using namespace cv;
 
namespace FacePreprocess {
 
    cv::Mat meanAxis0(const cv::Mat &src)
    {
        int num = src.rows;
        int dim = src.cols;
 
        // x1 y1
        // x2 y2
 
        cv::Mat output(1,dim,CV_32F);
        for(int i = 0 ; i <  dim; i ++)
        {
            float sum = 0 ;
            for(int j = 0 ; j < num ; j++)
            {
                sum+=src.at(j,i);
            }
            output.at(0,i) = sum/num;
        }
 
        return output;
    }
 
    cv::Mat elementwiseMinus(const cv::Mat &A,const cv::Mat &B)
    {
        cv::Mat output(A.rows,A.cols,A.type());
 
        assert(B.cols == A.cols);
        if(B.cols == A.cols)
        {
            for(int i = 0 ; i <  A.rows; i ++)
            {
                for(int j = 0 ; j < B.cols; j++)
                {
                    output.at(i,j) = A.at(i,j) - B.at(0,j);
                }
            }
        }
        return output;
    }
 
 
    cv::Mat varAxis0(const cv::Mat &src)
    {
        cv:Mat temp_ = elementwiseMinus(src,meanAxis0(src));
        cv::multiply(temp_ ,temp_ ,temp_ );
        return meanAxis0(temp_);
 
    }
 
 
 
    int MatrixRank(cv::Mat M)
    {
        Mat w, u, vt;
        SVD::compute(M, w, u, vt);
        Mat1b nonZeroSingularValues = w > 0.0001;
        int rank = countNonZero(nonZeroSingularValues);
        return rank;
 
    }
 
//    References
//    ----------
//    .. [1] "Least-squares estimation of transformation parameters between two
//    point patterns", Shinji Umeyama, PAMI 1991, DOI: 10.1109/34.88573
//
//    """
//
//    Anthor:Jack Yu
    cv::Mat similarTransform(cv::Mat src,cv::Mat dst) {
        int num = src.rows;
        int dim = src.cols;
        cv::Mat src_mean = meanAxis0(src);
        cv::Mat dst_mean = meanAxis0(dst);
        cv::Mat src_demean = elementwiseMinus(src, src_mean);
        cv::Mat dst_demean = elementwiseMinus(dst, dst_mean);
        cv::Mat A = (dst_demean.t() * src_demean) / static_cast(num);
        cv::Mat d(dim, 1, CV_32F);
        d.setTo(1.0f);
        if (cv::determinant(A) < 0) {
            d.at(dim - 1, 0) = -1;
 
        }
        Mat T = cv::Mat::eye(dim + 1, dim + 1, CV_32F);
        cv::Mat U, S, V;
        SVD::compute(A, S,U, V);
 
        // the SVD function in opencv differ from scipy .
 
 
        int rank = MatrixRank(A);
        if (rank == 0) {
            assert(rank == 0);
 
        } else if (rank == dim - 1) {
            if (cv::determinant(U) * cv::determinant(V) > 0) {
                T.rowRange(0, dim).colRange(0, dim) = U * V;
            } else {
//            s = d[dim - 1]
//            d[dim - 1] = -1
//            T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V))
//            d[dim - 1] = s
                int s = d.at(dim - 1, 0) = -1;
                d.at(dim - 1, 0) = -1;
 
                T.rowRange(0, dim).colRange(0, dim) = U * V;
                cv::Mat diag_ = cv::Mat::diag(d);
                cv::Mat twp = diag_*V; //np.dot(np.diag(d), V.T)
                Mat B = Mat::zeros(3, 3, CV_8UC1);
                Mat C = B.diag(0);
                T.rowRange(0, dim).colRange(0, dim) = U* twp;
                d.at(dim - 1, 0) = s;
            }
        }
        else{
            cv::Mat diag_ = cv::Mat::diag(d);
            cv::Mat twp = diag_*V.t(); //np.dot(np.diag(d), V.T)
            cv::Mat res = U* twp; // U
            T.rowRange(0, dim).colRange(0, dim) = -U.t()* twp;
        }
        cv::Mat var_ = varAxis0(src_demean);
        float val = cv::sum(var_).val[0];
        cv::Mat res;
        cv::multiply(d,S,res);
        float scale =  1.0/val*cv::sum(res).val[0];
        T.rowRange(0, dim).colRange(0, dim) = - T.rowRange(0, dim).colRange(0, dim).t();
        cv::Mat  temp1 = T.rowRange(0, dim).colRange(0, dim); // T[:dim, :dim]
        cv::Mat  temp2 = src_mean.t(); //src_mean.T
        cv::Mat  temp3 = temp1*temp2; // np.dot(T[:dim, :dim], src_mean.T)
        cv::Mat temp4 = scale*temp3;
        T.rowRange(0, dim).colRange(dim, dim+1)=  -(temp4 - dst_mean.t()) ;
        T.rowRange(0, dim).colRange(0, dim) *= scale;
        return T;
    }
 
 
}
#endif //FACE_DEMO_FACEPREPROCESS_H

2.2 main.cpp

#include
#include "FaceProprocess.h"

//标准的关键点。
float v1[5][2] = {
    {30.2946f, 51.6963f},
    {65.5318f, 51.5014f},
    {48.0252f, 71.7366f},
    {33.5493f, 92.3655f},
    {62.7299f, 92.2041f}};

//检测出的人脸框坐标和关键点
// orgimg.shape:  (640, 640, 3)
// x1:243.000000,y1:140.000000,x2:447.000000,y2:394.000000
// point_0_x:307.000000
// point_0_y:252.000000
// point_1_x:390.000000
// point_1_y:241.000000
// point_2_x:359.000000
// point_2_y:305.000000
// point_3_x:332.000000
// point_3_y:340.000000
// point_4_x:395.000000
// point_4_y:332.000000
//#define originWidth 640
//#define wantWidth   112

#define standardWidth 96
#define box_x1 243.00
#define box_y1 140.00
#define box_x2 447.00
#define box_y2 394.00
#define faceWidth   (box_x2 - box_x1)
#define faceHeight  (box_y2 - box_y1)
float point[10] = {307.00, 252.00, 390.00, 241.00, 359.00, 305.00, 332.00, 340.00, 395.00, 332.00};

// float v2[5][2] ={
//     {307.00, 252.00},
//     {390.00, 241.00},
//     {359.00, 305.00},
//     {332.00, 340.00},
//     {395.00, 332.00}};

float v2[5][2] ={
    {0.00, 0.00},
    {0.00, 0.00},
    {0.00, 0.00},
    {0.00, 0.00},
    {0.00, 0.00}};
    
int main(void)
{
    printf("faceWidth:%d\n", faceWidth);
    printf("faceHeight:%d\n", faceHeight);

    cv::Mat src(5,2,CV_32FC1, v1);
    memcpy(src.data, v1, 2 * 5 * sizeof(float));

    for (int j = 0; j<5; j = j + 1)
    {
        v2[j][0] = (point[2*j]     - box_x1)*(standardWidth/faceWidth);
        v2[j][1] = (point[2*j + 1] - box_y1)*(standardWidth/faceWidth);
        printf("v2[%d][0]:%f\n", j, v2[j][0]);
        printf("v2[%d][1]:%f\n", j, v2[j][1]);
    }

    cv::Mat dst(5,2,CV_32FC1, v2);

    memcpy(dst.data, v2, 2 * 5 * sizeof(float));
    cv::Mat m = FacePreprocess::similarTransform(dst ,src);

    cv::Mat map_matrix;
    cv::Rect map_matrix_r = Rect(0, 0, 3, 2); 
    cv::Mat (m, map_matrix_r).copyTo(map_matrix);

    cv::Mat croppedImageAligned;
    //第四个参数是图像的输出尺寸。

    cv::Mat srcImage = imread("./ldh.jpeg");
    cv::Mat croppedImage = srcImage(cv::Rect(box_x1, box_y1, faceWidth, faceHeight));// Rect(x,y,width,height)

    warpAffine(croppedImage, croppedImageAligned, map_matrix, { (int)faceWidth, (int)faceHeight}, CV_INTER_CUBIC | CV_WARP_FILL_OUTLIERS, BORDER_CONSTANT, cvScalarAll(0));
    cv::imwrite("./result.jpg", croppedImageAligned);
    return 1;
}

2.3 makefile

CC      = gcc
CPP     = g++
AR      = ar
RM      = rm -f
 
#Define the resource compiler.
RC = windres
 
## debug flag  
DBG_ENABLE := 1
 
OS = $(shell uname)
 
## source file path  
SRC_PATH  := ./src/
SRC_PATH_EXT := 
DEMO :=./demo/main.cpp
 
## target file name  
TARGET     := facealign
DEMO_TARGET := run
 
## get all source files  
SRCS := $(foreach spath, $(SRC_PATH), $(wildcard $(spath)*.c) $(wildcard $(spath)*.cpp))
 
## all .o based on all .c/.cpp
OBJS = $(SRCS:.c=.o)
OBJS := $(OBJS:.cpp=.o) 
 
## macro define
DEFS := __LINUX__ OS_LINUX _FILE_OFFSET_BITS=64
 
#if freeimage is static-linked use this !
#DEFS += FREEIMAGE_LIB
 
#LIBS   += -lopencv_calib3d  -lopencv_features2d -lopencv_flann  -l
#LIBS   += -l -lopencv_ml -lopencv_objdetect  -lopencv_legacy -lopencv_gpu
#LIBS   += -l   -lopencv_superres -lopencv_contrib -lopencv_nonfree
#LIBS   += -l  -l -lopencv_ts  -lopencv_stitching -lopencv_core


 
 
## need libs, add at here  
LIBS := opencv_core opencv_video opencv_highgui opencv_imgproc opencv_photo opencv_videostab opencv_highgui opencv_imgproc opencv_imgcodecs boost_system boost_thread 
 
DEMO_LIBS = $(LIBS)
DEMO_LIBS += facealign
 
## used headers  file path  
INCLUDE_PATH := ./include  ./src  /usr/include/
 
#$(warning $(INCLUDE_PATH))
 
## used include librarys file path  
LIBRARY_PATH := ./  ./libs /usr/lib/x86_64-linux-gnu/    
 
## debug for debug info, when use gdb to debug  
ifeq (1, ${DBG_ENABLE})   
CFLAGS += -D_DEBUG -g -DDEBUG=1 
else
CFLAGS += -O2 -DNDEBUG
endif
 
#for ENCYPT flags
 
ifeq ($(OS), Linux)
LIBS += dl rt
CFLAGS += -fPIC
TARGET_EXT := .so
LIBRARY_PATH += 
LDFLAGS += -Wl,--rpath=./libs 
endif
 
#CFLAGS += -msse4.2 -march=core2 -pipe $(foreach m, $(DEFS), -D$(m)) 
#CFLAGS += -march=armv8.2-a -pipe $(foreach m, $(DEFS), -D$(m)) 
 
CFLAGS += -pipe $(foreach m, $(DEFS), -D$(m)) 
  
## get all include path  
CFLAGS  += $(foreach dir, $(INCLUDE_PATH), -I$(dir))  
 
CXXFLAGS += $(CFLAGS) -std=c++11
 
## get all library path  
LDFLAGS += -lpthread $(foreach lib, $(LIBRARY_PATH), -L$(lib))
DEMO_LDFLAGS := $(LDFLAGS)
## get all librarys  
LDFLAGS += $(foreach lib, $(LIBS), -l$(lib))
 
DEMO_LDFLAGS += $(foreach lib, $(DEMO_LIBS), -l$(lib))
 
RCFLAGS ?= -DNDEBUG
 
 
default: all
 
%.o: %.c
	$(CC) $(CFLAGS) -g -c $< -o $@
 
%.o: %.cpp
	$(CPP) $(CXXFLAGS) -g -c $< -o $@
    
all: $(OBJS) $(RESOURCE)
    #$(CPP) $(CXXFLAGS) -o $(TARGET) $(OBJS) $(RESOURCE) $(LDFLAGS)
	$(CPP) $(CXXFLAGS) -g -shared -o lib$(TARGET)$(TARGET_EXT) $(OBJS) $(RESOURCE) $(LDFLAGS)
	mv libfacealign.so ./libs
	$(CPP) $(CXXFLAGS) -g -o $(DEMO_TARGET) $(DEMO) $(DEMO_LDFLAGS)
 
clean:  
	$(RM) $(OBJS) $(DEMO_TARGET) $(TARGET).* $(RESOURCE)

3 对齐效果

参考文献：

采用mtcnn输出的特征点做人脸对齐C++代码 - 代码先锋网

similarity transform matrix in c++ is different from python · Issue #481 · deepinsight/insightface · GitHub