静默活体检测+人脸检测+人脸识别结合在NCNN模型下的推理(Windows下的VS环境）
前言：
涉及到三个模型 静默活体检测模型<2M，人脸检测模型<2M ，人脸识别<5M(模型大小)
至于NCNN不必多说,全C++实现，不依赖第三方库实现，在第三方移动端CPU运行最快。
首先，这是三者结合的推理，这意味着从训练到转ncnn模型全部完成且作用效果的精度达到了不错的要求。
训练在此就省略了，复现的都是都是GitHub上的原项目，然后使用其最终模型进行C++上的推理。
实现的功能：
一个demo实现对单个人的识别。（因为未涉及C++的太多知识，比如通过序列化对象转储到文件并从文件中读取，或者是通过连接数据库进行数据的转储，所以测试的推理的整个流程针对单人。）
步骤:先进行静默活体检测->人脸检测->人脸识别。后面的步骤都是在前面的步骤上实现，识别完后进行静默活体检测开始循环反复。
main函数
#include
#include
#include
#include "platform.h"
#include
#include
#include
#include "iostream"
#include
#include "net.h"
#if NCNN_VULKAN
#include "gpu.h"
#endif // NCNN_VULKAN
#define IMAGE_SIZE 80
#define CV_RGB(r,g,b) cvScalar((b),(g),(r),0)
/*声明*/
bool JudgeReal(cv::Mat bgr, ncnn::Extractor ex, ncnn::Net & SilentFaceSpoofing);
cv::Mat GetMyMat(cv::Mat img, const std::string& s);//获取人脸特征
cv::Mat GetMat(const std::string & path);//获取人脸框
cv::Mat GetMat(cv::Mat img);//重载
float* getFeatByMobileFaceNetNCNN(cv::Mat img);//获取特征向量
float CalcSimilarity_1(float* fc1,//比较出相识度
float* fc2,
long dim);
std::string Convert(float Num)
{
std:: ostringstream oss;
oss << Num;
std:: string str(oss.str());
return str;
}
int main(){
/*
先加载一个人的图片及其特征向量
*/
cv::Mat sample = GetMat("./pic/yzh1.jpg");
float *sam= getFeatByMobileFaceNetNCNN( sample);
/*
加载活体检测模型
*/
static ncnn::Net SilentFaceSpoofing;
SilentFaceSpoofing.load_param("./model/2.7_80x80_MiniFASNetV2_sim.param");
SilentFaceSpoofing.load_model("./model/2.7_80x80_MiniFASNetV2_sim.bin");
static ncnn::Extractor ex = SilentFaceSpoofing.create_extractor();
//打开摄像头
cv::VideoCapture capture(0);//创建VideoCapture对象
if (!capture.isOpened())//判断是否打开摄像头，打开isOpened返回ture
return 1;
bool stop(false);//定义一个用来停止循环的变量
cv::Mat frame;//用来存放读取的视频序列，承载每一帧的图像 ，Mat类是用于保存图像以及其他矩阵数据的数据结构
cv::namedWindow("Camera");//创建一个窗口，显示每一帧的窗口
while (!stop)
{
if (!capture.read(frame))//如果没有读取到就中断
{
break;
}
bool result;
result = JudgeReal(frame, ex, SilentFaceSpoofing);//判断真假脸
if (result) {//活体
cv::Mat frame2= GetMat(frame);//获取人脸框
float * fr= getFeatByMobileFaceNetNCNN(frame2);
float res= CalcSimilarity_1(fr, sam, 112);
std::cout << "相识度:" << res << std::endl;
delete[]fr;
if (res > 0.3) { //认为是同一个人
std::string name = "yzh-sim:" + Convert(res);
cv::Mat imgs = GetMyMat(frame, name);
putText(imgs, "realface", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4);
cv::imshow("Camera", imgs);//正常显示，把获取的视频填充到窗口中
}
else
{
cv::Mat imgs = GetMyMat(frame, "unknow");
putText(imgs, "unkonwrealface", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4);
cv::imshow("Camera", imgs);//正常显示，把获取的视频填充到窗口中
}
}
else {
putText(frame, "FalseFace", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4);
cv::imshow("Camera", frame);//正常显示，把获取的视频填充到窗口中
std::cout << result << std::endl;
}
char c = cvWaitKey(33);
if (c == 32)break; //使用空格键来停止ASCII 为32 关闭摄像头
}
capture.release();//释放
return 0;
}
获取人的特征向量以及比较人的相识度（mobilefacenet网络实现识别）
#include
#include "net.h"
#include
using namespace std;
static int flag = 0;
//Mobilefacenet 初始化
float simd_dot_1(const float* x, const float* y, const long& len) {
float inner_prod = 0.0f;
__m128 X, Y; // 128-bit values
__m128 acc = _mm_setzero_ps(); // set to (0, 0, 0, 0)
float temp[4];
long i;
for (i = 0; i + 4 < len; i += 4) {
X = _mm_loadu_ps(x + i); // load chunk of 4 floats
Y = _mm_loadu_ps(y + i);
acc = _mm_add_ps(acc, _mm_mul_ps(X, Y));
}
_mm_storeu_ps(&temp[0], acc); // store acc into an array
inner_prod = temp[0] + temp[1] + temp[2] + temp[3];
// add the remaining values
for (; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
}
float CalcSimilarity_1( float* fc1,
float* fc2,
long dim) {
return simd_dot_1(fc1, fc2, dim)
/ (sqrt(simd_dot_1(fc1, fc1, dim))
* sqrt(simd_dot_1(fc2, fc2, dim)));
}
float* getFeatByMobileFaceNetNCNN( cv::Mat img)
{
static ncnn::Net Recognet;
if (!flag) {
Recognet.load_param("./model/mobilefacenet.param");
Recognet.load_model("./model/mobilefacenet.bin");
}
ncnn::Extractor ex= Recognet.create_extractor();
ex.set_light_mode(true);
ex.set_num_threads(4);
//cout << "getFeatByMobileFaceNetNCNN" << endl;
float* feat = new float[512];
ncnn::Mat in = ncnn::Mat::from_pixels_resize(img.data, ncnn::Mat::PIXEL_BGR, img.cols, img.rows, 112, 112);
ex.input("data", in);
ncnn::Mat out;
ex.extract("fc1", out);
//std::vector cls_scores;
for (int j = 0; j < out.w; j++)
{
feat[j] = out[j];
}
return feat;
} 静默活体识别引用函数（实现预处理）
#include
#include
#include
#include
#include
#include "iostream"
#include "platform.h"
#include "net.h"
#if NCNN_VULKAN
#include "gpu.h"
#endif // NCNN_VULKAN
#define IMAGE_SIZE 80
bool JudgeReal(cv::Mat bgr, ncnn::Extractor ex, ncnn::Net& SilentFaceSpoofing) {
std::vector cls_scores;
ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, IMAGE_SIZE, IMAGE_SIZE);
fprintf(stderr, "input shape: %d %d %d %d\n", in.dims, in.h, in.w, in.c);
ex.input("input", in);//input 是 .param文件中输入节点名称
ncnn::Mat out;
ex.extract("output", out);
{
ncnn::Layer* softmax = ncnn::create_layer("Softmax");
ncnn::ParamDict pb;
softmax->load_param(pb);
softmax->forward_inplace(out, SilentFaceSpoofing.opt);
delete softmax;
}
out = out.reshape(out.h * out.w * out.c);
fprintf(stderr, "output shape: %d %d %d %d\n", out.dims, out.h, out.w, out.c);
cls_scores.resize(out.w);
for (int j = 0; j < out.w; j++)
{
cls_scores[j] = out[j];
printf("cls_scores[%d]=%f\n", j, cls_scores[j]);
}
std::cout << std::endl;
auto itMax = max_element(cls_scores.begin(), cls_scores.end());
std::cout << "the max:" << *itMax << " the location:" << distance(cls_scores.begin(), itMax) << std::endl;
if (distance(cls_scores.begin(), itMax) == 1) {
std::cout << "Image " << distance(cls_scores.begin(), itMax) << " is Real Face. Score: " << *itMax << std::endl;
return true;
}
else {
std::cout << "Image " << distance(cls_scores.begin(), itMax) << " is Fake Face. Score: " << *itMax << std::endl;
return false;
}
}
获取图像中的人脸框及其特征点 （retinaface网络实现检测，这里的boxsize是图片中检测的人脸个数，由于本次demo只针对单人，所以返回的总是最后一个人的检测框）
#include
#include
#include
#include
#include
#include
#include
#include "FaceDetector.h"
using namespace std;
static bbox mybox;
//cv::Mat GetMat()
/*
图像预处理
获取框选后的人脸图像
*/
cv::Mat GetMat(const std::string & path)
{
string imgPath = path;
static string param = "./model/face.param";
static string bin = "./model/face.bin";
static const int max_side = 320;
static Detector detector(param, bin, false);
// retinaface
// Detector detector(param, bin, true);
Timer timer;
cv::Mat img = cv::imread(imgPath.c_str(), CV_LOAD_IMAGE_COLOR);
// scale
float long_side = max(img.cols, img.rows);
float scale = max_side/long_side;
cv::Mat img_scale;
cv::Size size = cv::Size(img.cols*scale, img.rows*scale);
cv::resize(img, img_scale, cv::Size(img.cols*scale, img.rows*scale));
if (img.empty())
{
fprintf(stderr, "cv::imread %s failed\n", imgPath.c_str());
}
std::vector boxes;
timer.tic();
detector.Detect(img_scale, boxes);
timer.toc("----total timer:");
if (!boxes.size()) {
cv::Mat temp;
cv::resize(img, temp, cv::Size(112, 112));
return temp;
}
cv::Mat temps;
// draw image 裁剪出图片中人脸框
for (int j = 0; j < boxes.size(); ++j) {
cv::Rect rect(boxes[j].x1 / scale, boxes[j].y1 / scale, boxes[j].x2 / scale - boxes[j].x1 / scale, boxes[j].y2 / scale - boxes[j].y1 / scale);
cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 1, 8, 0);
// char test[80];
cv::Mat m = img(rect);
cv::resize(m, temps, cv::Size(112, 112));
mybox = boxes[j];
}
return temps;
}
cv::Mat GetMat( cv::Mat img)
{
static string param = "./model/face.param";
static string bin = "./model/face.bin";
static const int max_side = 320;
static Detector detector(param, bin, false);
// retinaface
// Detector detector(param, bin, true);
Timer timer;
// scale
float long_side = max(img.cols, img.rows);
float scale = max_side / long_side;
cv::Mat img_scale;
cv::Size size = cv::Size(img.cols * scale, img.rows * scale);
cv::resize(img, img_scale, cv::Size(img.cols * scale, img.rows * scale));
if (img.empty())
{
std::cout << "empty img" << std::endl;
}
std::vector boxes;
timer.tic();
detector.Detect(img_scale, boxes);
timer.toc("----total timer:");
if (!boxes.size()) {
cv::Mat temp;
cv::resize(img, temp, cv::Size(112, 112));
return temp;
}
cv::Mat temps;
// draw image 裁剪出图片中人脸框
for (int j = 0; j < boxes.size(); ++j) {
cv::Rect rect(boxes[j].x1 / scale, boxes[j].y1 / scale, boxes[j].x2 / scale - boxes[j].x1 / scale, boxes[j].y2 / scale - boxes[j].y1 / scale);
cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 1, 8, 0);
// char test[80];
cv::Mat m = img(rect);
cv::resize(m, temps, cv::Size(112, 112));
mybox = boxes[j];
}
return temps;
}
/*
获取图像框
*/
cv::Mat GetMyMat( cv::Mat img, const std::string &s) {
float long_side = max(img.cols, img.rows);
float scale = 320 / long_side;
cv::Rect rect(mybox.x1 / scale, mybox.y1 / scale, mybox.x2 / scale - mybox.x1 / scale, mybox.y2 / scale - mybox.y1 / scale);
cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 3, 8, 0);
cv::putText(img, s, cv::Size((mybox.x1 / scale), mybox.y1 / scale), cv::FONT_HERSHEY_COMPLEX, 0.5, cv::Scalar(0, 255, 255));
cv::circle(img, cv::Point(mybox.point[0]._x / scale, mybox.point[0]._y / scale), 1, cv::Scalar(0, 0, 225), 4);
cv::circle(img, cv::Point(mybox.point[1]._x / scale, mybox.point[1]._y / scale), 1, cv::Scalar(0, 255, 225), 4);
cv::circle(img, cv::Point(mybox.point[2]._x / scale, mybox.point[2]._y / scale), 1, cv::Scalar(255, 0, 225), 4);
cv::circle(img, cv::Point(mybox.point[3]._x / scale, mybox.point[3]._y / scale), 1, cv::Scalar(0, 255, 0), 4);
cv::circle(img, cv::Point(mybox.point[4]._x / scale, mybox.point[4]._y / scale), 1, cv::Scalar(255, 0, 0), 4);
return img;
}
#include
//#include "omp.h"
#include "FaceDetector.h"
using namespace std;
Detector::Detector():
_nms(0.4),
_threshold(0.6),
_mean_val{104.f, 117.f, 123.f},
_retinaface(false),
Net(new ncnn::Net())
{
}
inline void Detector::Release(){
if (Net != nullptr)
{
delete Net;
Net = nullptr;
}
}
Detector::Detector(const std::string &model_param, const std::string &model_bin, bool retinaface):
_nms(0.4),
_threshold(0.6),
_mean_val{104.f, 117.f, 123.f},
_retinaface(retinaface),
Net(new ncnn::Net())
{
Init(model_param, model_bin);
}
void Detector::Init(const std::string &model_param, const std::string &model_bin)
{
int ret = Net->load_param(model_param.c_str());
ret = Net->load_model(model_bin.c_str());
}
void Detector::Detect(cv::Mat& bgr, std::vector& boxes)
{
Timer timer;
timer.tic();
ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, bgr.cols, bgr.rows);
in.substract_mean_normalize(_mean_val, 0);
timer.toc("precoss:");
timer.tic();
ncnn::Extractor ex = Net->create_extractor();
ex.set_light_mode(true);
ex.set_num_threads(4);
ex.input(0, in);
ncnn::Mat out, out1, out2;
// loc
ex.extract("output0", out);
// class
ex.extract("530", out1);
//landmark
ex.extract("529", out2);
timer.toc("det:");
std::vector anchor;
timer.tic();
if (_retinaface)
create_anchor_retinaface(anchor, bgr.cols, bgr.rows);
else
create_anchor(anchor, bgr.cols, bgr.rows);
timer.toc("anchor:");
std::vector total_box;
float *ptr = out.channel(0);
float *ptr1 = out1.channel(0);
float *landms = out2.channel(0);
// #pragma omp parallel for num_threads(2)
for (int i = 0; i < anchor.size(); ++i)
{
if (*(ptr1+1) > _threshold)
{
box tmp = anchor[i];
box tmp1;
bbox result;
// loc and conf
tmp1.cx = tmp.cx + *ptr * 0.1 * tmp.sx;
tmp1.cy = tmp.cy + *(ptr+1) * 0.1 * tmp.sy;
tmp1.sx = tmp.sx * exp(*(ptr+2) * 0.2);
tmp1.sy = tmp.sy * exp(*(ptr+3) * 0.2);
result.x1 = (tmp1.cx - tmp1.sx/2) * in.w;
if (result.x1<0)
result.x1 = 0;
result.y1 = (tmp1.cy - tmp1.sy/2) * in.h;
if (result.y1<0)
result.y1 = 0;
result.x2 = (tmp1.cx + tmp1.sx/2) * in.w;
if (result.x2>in.w)
result.x2 = in.w;
result.y2 = (tmp1.cy + tmp1.sy/2)* in.h;
if (result.y2>in.h)
result.y2 = in.h;
result.s = *(ptr1 + 1);
// landmark
for (int j = 0; j < 5; ++j)
{
result.point[j]._x =( tmp.cx + *(landms + (j<<1)) * 0.1 * tmp.sx ) * in.w;
result.point[j]._y =( tmp.cy + *(landms + (j<<1) + 1) * 0.1 * tmp.sy ) * in.h;
}
total_box.push_back(result);
}
ptr += 4;
ptr1 += 2;
landms += 10;
}
std::sort(total_box.begin(), total_box.end(), cmp);
nms(total_box, _nms);
printf("%d\n", (int)total_box.size());
for (int j = 0; j < total_box.size(); ++j)
{
boxes.push_back(total_box[j]);
}
}
inline bool Detector::cmp(bbox a, bbox b) {
if (a.s > b.s)
return true;
return false;
}
inline void Detector::SetDefaultParams(){
_nms = 0.4;
_threshold = 0.6;
_mean_val[0] = 104;
_mean_val[1] = 117;
_mean_val[2] = 123;
Net = nullptr;
}
Detector::~Detector(){
Release();
}
void Detector::create_anchor(std::vector &anchor, int w, int h)
{
// anchor.reserve(num_boxes);
anchor.clear();
std::vector > feature_map(4), min_sizes(4);
float steps[] = {8, 16, 32, 64};
for (int i = 0; i < feature_map.size(); ++i) {
feature_map[i].push_back(ceil(h/steps[i]));
feature_map[i].push_back(ceil(w/steps[i]));
}
std::vector minsize1 = {10, 16, 24};
min_sizes[0] = minsize1;
std::vector minsize2 = {32, 48};
min_sizes[1] = minsize2;
std::vector minsize3 = {64, 96};
min_sizes[2] = minsize3;
std::vector minsize4 = {128, 192, 256};
min_sizes[3] = minsize4;
for (int k = 0; k < feature_map.size(); ++k)
{
std::vector min_size = min_sizes[k];
for (int i = 0; i < feature_map[k][0]; ++i)
{
for (int j = 0; j < feature_map[k][1]; ++j)
{
for (int l = 0; l < min_size.size(); ++l)
{
float s_kx = min_size[l]*1.0/w;
float s_ky = min_size[l]*1.0/h;
float cx = (j + 0.5) * steps[k]/w;
float cy = (i + 0.5) * steps[k]/h;
box axil = {cx, cy, s_kx, s_ky};
anchor.push_back(axil);
}
}
}
}
}
void Detector::create_anchor_retinaface(std::vector &anchor, int w, int h)
{
// anchor.reserve(num_boxes);
anchor.clear();
std::vector > feature_map(3), min_sizes(3);
float steps[] = {8, 16, 32};
for (int i = 0; i < feature_map.size(); ++i) {
feature_map[i].push_back(ceil(h/steps[i]));
feature_map[i].push_back(ceil(w/steps[i]));
}
std::vector minsize1 = {10, 20};
min_sizes[0] = minsize1;
std::vector minsize2 = {32, 64};
min_sizes[1] = minsize2;
std::vector minsize3 = {128, 256};
min_sizes[2] = minsize3;
for (int k = 0; k < feature_map.size(); ++k)
{
std::vector min_size = min_sizes[k];
for (int i = 0; i < feature_map[k][0]; ++i)
{
for (int j = 0; j < feature_map[k][1]; ++j)
{
for (int l = 0; l < min_size.size(); ++l)
{
float s_kx = min_size[l]*1.0/w;
float s_ky = min_size[l]*1.0/h;
float cx = (j + 0.5) * steps[k]/w;
float cy = (i + 0.5) * steps[k]/h;
box axil = {cx, cy, s_kx, s_ky};
anchor.push_back(axil);
}
}
}
}
}
void Detector::nms(std::vector &input_boxes, float NMS_THRESH)
{
std::vectorvArea(input_boxes.size());
for (int i = 0; i < int(input_boxes.size()); ++i)
{
vArea[i] = (input_boxes.at(i).x2 - input_boxes.at(i).x1 + 1)
* (input_boxes.at(i).y2 - input_boxes.at(i).y1 + 1);
}
for (int i = 0; i < int(input_boxes.size()); ++i)
{
for (int j = i + 1; j < int(input_boxes.size());)
{
float xx1 = max(input_boxes[i].x1, input_boxes[j].x1);
float yy1 = max(input_boxes[i].y1, input_boxes[j].y1);
float xx2 = min(input_boxes[i].x2, input_boxes[j].x2);
float yy2 = min(input_boxes[i].y2, input_boxes[j].y2);
float w = max(float(0), xx2 - xx1 + 1);
float h = max(float(0), yy2 - yy1 + 1);
float inter = w * h;
float ovr = inter / (vArea[i] + vArea[j] - inter);
if (ovr >= NMS_THRESH)
{
input_boxes.erase(input_boxes.begin() + j);
vArea.erase(vArea.begin() + j);
}
else
{
j++;
}
}
}
}
效果:
真脸
最后:
静默活体检测还存在一定误差。感谢各位大佬的开源，让作为菜鸟的我完成了缝合。
参考:
38、静默活体检测测试及ncnn、mnn部署_sxj731533730的博客-CSDN博客_ncnn和mnn
deepinsight/insightface: State-of-the-art 2D and 3D Face Analysis Project (github.com)
GitHub - biubug6/Pytorch_Retinaface: Retinaface get 80.99% in widerface hard val using mobilenet0.25.