Opencv C++教学视频源代码整理（2）
P132 CNN模型预测年龄与性别
(1)、CSDN上的CNN模型预测年龄与性别代码1：成功，效果一般
#include
#include
#include
using namespace cv;
using namespace cv::dnn;
using namespace std;
//人脸检测文件
String haar_file = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/haarcascade_frontalface_alt_tree.xml";
//年龄预测模型
String age_model = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/age_net.caffemodel";
//年龄描述文件
String age_text = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/deploy_age.prototxt";
//性别预测模型
String gender_model = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/gender_net.caffemodel";
//年龄描述文件
String gender_text = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/deploy_gender.prototxt";
void predict_age(Net &net, Mat &image);//预测年龄
void predict_gender(Net &net, Mat &image);//预测性别
int main(int argc, char** argv) {
Mat src = imread("C:/Users/25503/Desktop/樊玉和.jpg");
if (src.empty()) {
printf("could not load image...\n");
return -1;
}
namedWindow("input", CV_WINDOW_AUTOSIZE);
imshow("input", src);
CascadeClassifier detector;
detector.load(haar_file);//人脸检测
vector
Mat gray;
cvtColor(src, gray, COLOR_BGR2GRAY);
detector.detectMultiScale(gray, faces, 1.02, 1, 0, Size(40, 40), Size(200, 200));
//加载网络
Net age_net = readNetFromCaffe(age_text, age_model);
Net gender_net = readNetFromCaffe(gender_text, gender_model);
for (size_t t = 0; t < faces.size(); t++) {
rectangle(src, faces[t], Scalar(30, 255, 30), 2, 8, 0);
//年龄、性别预测
Mat face = src(faces[t]);//自己加的，不加会报错，提示类型错误
predict_age(age_net, face);
predict_gender(age_net, face);
}
imshow("age-gender-prediction-demo", src);
waitKey(0);
return 0;
}
vector
vector
ages.push_back("0-2");
ages.push_back("4 - 6");
ages.push_back("8 - 13");
ages.push_back("15 - 20");
ages.push_back("25 - 32");
ages.push_back("38 - 43");
ages.push_back("48 - 53");
ages.push_back("60-");
return ages;
}
void predict_age(Net &net, Mat &image) {
// 输入
Mat blob = blobFromImage(image, 1.0, Size(227, 227));
net.setInput(blob, "data");
// 预测分类
Mat prob = net.forward("prob");
Mat probMat = prob.reshape(1, 1);//变为一行
Point classNum;
double classProb;
vector
minMaxLoc(probMat, NULL, &classProb, NULL, &classNum);//提取最大概率的编号和概率值
int classidx = classNum.x;
putText(image, format("age:%s", ages.at(classidx).c_str()), Point(1, 10), FONT_HERSHEY_PLAIN, 0.8, Scalar(0, 0, 255), 1);
}
void predict_gender(Net &net, Mat &image) {
// 输入
Mat blob = blobFromImage(image, 1.0, Size(227, 227));
net.setInput(blob, "data");
// 预测分类
Mat prob = net.forward("prob");
Mat probMat = prob.reshape(1, 1);
putText(image, format("gender:%s", (probMat.at
Point(2, 20), FONT_HERSHEY_PLAIN, 0.8, Scalar(0, 0, 255), 1);
}
(2)、教学视频上的 CNN模型预测年龄与性别代码：一般，教学视频代码可能不行，需要OPenCV 3.3版本
#include
#include
#include
using namespace cv;
using namespace cv::dnn;
using namespace std;
String haar_file = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/haarcascade_frontalface_alt_tree.xml";//人脸检测文件
String age_model = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/age_net.caffemodel";
String age_text = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/deploy_age.prototxt";
String gender_model = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/gender_net.caffemodel";
String gender_text = "D:/OpenCV神经网络2/CNN级联分类器检测人脸预测年龄性别/deploy_gender.prototxt";
void predict_age(Net &net, Mat &image);
void predict_gender(Net &net, Mat &image);
int main(int argc, char** argv) {
Mat src = imread("C:/Users/25503/Desktop/12321.jpg");
if (src.empty()) {
printf("could not load image...\n");
return -1;
}
namedWindow("input", CV_WINDOW_AUTOSIZE);
imshow("input", src);
CascadeClassifier detector;
detector.load(haar_file);
vector
Mat gray;
cvtColor(src, gray, COLOR_BGR2GRAY);
detector.detectMultiScale(gray, faces, 1.02, 1, 0, Size(40, 40), Size(200, 200));
Net age_net = readNetFromCaffe(age_text, age_model);
Net gender_net = readNetFromCaffe(gender_text, gender_model);
for (size_t t = 0; t < faces.size(); t++) {
rectangle(src, faces[t], Scalar(0, 0, 255), 2, 8, 0);
Mat face = src(faces[t]);//自己加的，不加会报错，提示类型错误
predict_age(age_net, face);
predict_gender(gender_net, face);
}
imshow("age-gender-prediction-demo", src);
waitKey(0);
return 0;
}
vector
vector
ages.push_back("0-2");
ages.push_back("4 - 6");
ages.push_back("8 - 13");
ages.push_back("15 - 20");
ages.push_back("25 - 32");
ages.push_back("38 - 43");
ages.push_back("48 - 53");
ages.push_back("60-");
return ages;
}
void predict_age(Net &net, Mat &image) {
// 输入
Mat blob = blobFromImage(image, 1.0, Size(227, 227));
net.setInput(blob, "data");
// 预测分类
Mat prob = net.forward("prob");
Mat probMat = prob.reshape(1, 1);
Point classNum;
double classProb;
vector
minMaxLoc(probMat, NULL, &classProb, NULL, &classNum);
int classidx = classNum.x;
putText(image, format("age:%s", ages.at(classidx).c_str()), Point(2, 10), FONT_HERSHEY_PLAIN, 0.8, Scalar(0, 0, 255), 1);
}
void predict_gender(Net &net, Mat &image) {
// 输入
Mat blob = blobFromImage(image, 1.0, Size(227, 227));
net.setInput(blob, "data");
// 预测分类
Mat prob = net.forward("prob");
Mat probMat = prob.reshape(1, 1);
putText(image, format("gender:%s", (probMat.at
Point(2, 20), FONT_HERSHEY_PLAIN, 0.8, Scalar(255, 0, 0), 1);
}
P133 GOTURN模型实现视频对象跟踪***失败，但需要时可以调试或者安装opencv 3.3.0版本
String goturn_model = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.caffemodel";
String goturn_prototxt = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.prototxt";
(1)、CSDN上的GOTURN模型实现视频对象跟踪代码：（失败）
#include
#include
#include
using namespace cv;
using namespace cv::dnn;
using namespace std;
String goturn_model = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.caffemodel";
String goturn_prototxt = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.prototxt";
Net net;
Rect trackObjects(Mat &frame, Mat &prevFrame);
Mat frame, prevFrame;
Rect prevBB;
int main(int argc, char** argv) {
net = readNetFromCaffe(goturn_prototxt, goturn_model);
//VideoCapture capture;
//capture.open("D:/test/dog.avi");
VideoCapture capture(0);
capture.read(frame);
frame.copyTo(prevFrame);
prevBB = selectROI(frame, true, true);
namedWindow("frame", WINDOW_AUTOSIZE);
while (capture.read(frame)) {
Rect currentBB = trackObjects(frame, prevFrame);
rectangle(frame, currentBB, Scalar(0, 0, 255), 2, 8, 0);
// ready for next frame
frame.copyTo(prevFrame);
prevBB.x = currentBB.x;
prevBB.y = currentBB.y;
prevBB.width = currentBB.width;
prevBB.height = currentBB.height;
imshow("frame", frame);
char c = waitKey(10);
if (c == 27) {
break;
}
}
}
Rect trackObjects(Mat& frame, Mat &prevFrame) {
Rect rect;
int INPUT_SIZE = 227;
//Using prevFrame & prevBB from model and curFrame GOTURN calculating curBB
Mat curFrame = frame.clone();
Rect2d curBB;
float padTargetPatch = 2.0;
Rect2f searchPatchRect, targetPatchRect;
Point2f currCenter, prevCenter;
Mat prevFramePadded, curFramePadded;
Mat searchPatch, targetPatch;
prevCenter.x = (float)(prevBB.x + prevBB.width / 2);
prevCenter.y = (float)(prevBB.y + prevBB.height / 2);
targetPatchRect.width = (float)(prevBB.width*padTargetPatch);
targetPatchRect.height = (float)(prevBB.height*padTargetPatch);
targetPatchRect.x = (float)(prevCenter.x - prevBB.width*padTargetPatch / 2.0 + targetPatchRect.width);
targetPatchRect.y = (float)(prevCenter.y - prevBB.height*padTargetPatch / 2.0 + targetPatchRect.height);
copyMakeBorder(prevFrame, prevFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE);
targetPatch = prevFramePadded(targetPatchRect).clone();
copyMakeBorder(curFrame, curFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE);
searchPatch = curFramePadded(targetPatchRect).clone();
//Preprocess
//Resize
resize(targetPatch, targetPatch, Size(INPUT_SIZE, INPUT_SIZE));
resize(searchPatch, searchPatch, Size(INPUT_SIZE, INPUT_SIZE));
//Mean Subtract
targetPatch = targetPatch - 128;
searchPatch = searchPatch - 128;
//Convert to Float type
targetPatch.convertTo(targetPatch, CV_32F);
searchPatch.convertTo(searchPatch, CV_32F);
Mat targetBlob = blobFromImage(targetPatch);
Mat searchBlob = blobFromImage(searchPatch);
net.setInput(targetBlob, "data1");
net.setInput(searchBlob, "data2");
Mat res = net.forward("scale");
Mat resMat = res.reshape(1, 1);
//printf("width : %d, height : %d\n", (resMat.at
curBB.x = targetPatchRect.x + (resMat.at
curBB.y = targetPatchRect.y + (resMat.at
curBB.width = (resMat.at
curBB.height = (resMat.at
//Predicted BB
Rect boundingBox = curBB;
return boundingBox;
}
(2)、教学视频上的 GOTURN模型实现视频对象跟踪代码：失败，因为部分语句关键词必须用opencv 3.3.0版本才可以，但学习时用的事3.4.9版本
#include
#include
#include
using namespace cv;
using namespace cv::dnn;
using namespace std;
String goturn_model = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.caffemodel";
String goturn_prototxt = "D:/OpenCV神经网络2/GOTURN算法模型实现视频对象跟踪/goturn.prototxt";
Net net;
void initGoturn();
Rect trackObjects(Mat& frame, Mat& prevFrame);
Mat frame, prevFrame;
Rect prevBB;
int main(int argc, char** argv) {
initGoturn();
//VideoCapture capture;
//capture.open("01.mp4");
VideoCapture capture(0);
capture.read(frame);
frame.copyTo(prevFrame);
prevBB = selectROI(frame, true, true);
namedWindow("frame", CV_WINDOW_AUTOSIZE);
while (capture.read(frame)) {
Rect currentBB = trackObjects(frame, prevFrame);
rectangle(frame, currentBB, Scalar(0, 0, 255), 2, 8, 0);
// ready for next frame
frame.copyTo(prevFrame);
prevBB.x = currentBB.x;
prevBB.y = currentBB.y;
prevBB.width = currentBB.width;
prevBB.height = currentBB.height;
imshow("frame", frame);
char c = waitKey(50);
if (c == 27) {
break;
}
}
}
void initGoturn() {// 只有opencv 3.3.0版本才存在Importer关键词，但学习用的是3.4.9版本，故用到此模块需要重新安装opencv 3.3.0版本
Ptr
importer = createCaffeImporter(goturn_prototxt, goturn_model);
importer->populateNet(net);
importer.release();
}
Rect trackObjects(Mat& frame, Mat& prevFrame) {
Rect rect;
int INPUT_SIZE = 227;
//Using prevFrame & prevBB from model and curFrame GOTURN calculating curBB
Mat curFrame = frame.clone();
Rect2d curBB;
float padTargetPatch = 2.0;
Rect2f searchPatchRect, targetPatchRect;
Point2f currCenter, prevCenter;
Mat prevFramePadded, curFramePadded;
Mat searchPatch, targetPatch;
prevCenter.x = (float)(prevBB.x + prevBB.width / 2);
prevCenter.y = (float)(prevBB.y + prevBB.height / 2);
targetPatchRect.width = (float)(prevBB.width * padTargetPatch);
targetPatchRect.height = (float)(prevBB.height * padTargetPatch);
targetPatchRect.x = (float)(prevCenter.x - prevBB.width * padTargetPatch / 2.0 + targetPatchRect.width);
targetPatchRect.y = (float)(prevCenter.y - prevBB.height * padTargetPatch / 2.0 + targetPatchRect.height);
copyMakeBorder(prevFrame, prevFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE);
targetPatch = prevFramePadded(targetPatchRect).clone();
copyMakeBorder(curFrame, curFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE);
searchPatch = curFramePadded(targetPatchRect).clone();
//Preprocess
//Resize
resize(targetPatch, targetPatch, Size(INPUT_SIZE, INPUT_SIZE));
resize(searchPatch, searchPatch, Size(INPUT_SIZE, INPUT_SIZE));
//Mean Subtract
targetPatch = targetPatch - 128;
searchPatch = searchPatch - 128;
//Convert to Float type
targetPatch.convertTo(targetPatch, CV_32F);
searchPatch.convertTo(searchPatch, CV_32F);
Mat targetBlob = blobFromImage(targetPatch);
Mat searchBlob = blobFromImage(searchPatch);
net.setInput(targetBlob, ".data1");
net.setInput(searchBlob, ".data2");
Mat res = net.forward("scale");
Mat resMat = res.reshape(1, 1);
//printf("width : %d, height : %d\n", (resMat.at
curBB.x = targetPatchRect.x + (resMat.at
curBB.y = targetPatchRect.y + (resMat.at
curBB.width = (resMat.at
curBB.height = (resMat.at
//Predicted BB
Rect boundingBox = curBB;
return boundingBox;
}
P134-P136人脸识别理论基础：概述、均值，协方差，协方差矩阵，特征值，特征向量
（1）图像均值与方差、方差与协方差矩阵代码：
#include
#include
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
Mat image = imread("C:/Users/25503/Desktop/999.png");
if (image.empty())
{
printf("could not load image...\n");
return -1;
}
imshow("input image", image);
Mat means, stddev;//均值和方差
meanStdDev(image, means, stddev);
printf("means rows:%d, means cols %d\n", means.rows, means.cols);
printf("stddev rows:%d,stddev cols %d\n", stddev.rows, stddev.cols);
for (int row = 0; row < means.rows; row++)
{
printf("means %d =%.3f\n", row, means.at
printf("stddev %d =%.3f\n", row, stddev.at
}
Mat samples = (Mat_
Mat cov, mu;//协方差和协方差矩阵
calcCovarMatrix(samples, cov, mu, CV_COVAR_NORMAL | CV_COVAR_ROWS);
cout << "==============================" << endl;
cout << "cov:" << endl;
cout << cov << endl;
cout << "means" << endl;
cout << mu << endl;
waitKey(0);
return 0;
}
(2) 特征值与特征向量矩阵的代码：
#include
#include
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
Mat data = (Mat_
1,2,
2, 1);
Mat eigenvalues, eigenvector;//特征值和特征向量矩阵
eigen(data, eigenvalues, eigenvector);//eigen必须要求的矩阵数据是对阵矩阵
for (int i = 0; i < eigenvalues.rows; i++)
{
printf("eigen value %d :%.3f\n", i, eigenvalues.at
}
cout << "eigen vector :" << endl;
cout << eigenvector << endl;
waitKey(0);
return 0;
}
P137-P138 PCA原理与应用：PCA是将高维数据降低到低位数据
（1）CSDN上的PCA原理代码：
#include
#include
using namespace cv;
using namespace std;
double calcPCAorientation(Mat &image, vector
int main(int argc, char** argv)
{
Mat src = imread("D:/opencv3.4.9/opencv/sources/samples/data/pca_test1.jpg");
if (src.empty())
{
cout << "图片未找到..." << endl;
return -1;
}
imshow("input image", src);
Mat gray, binary;
//转为灰度图
cvtColor(src, gray, COLOR_BGR2GRAY);
//转为二值图
threshold(gray, binary, 0, 255, THRESH_BINARY | THRESH_OTSU);//自动获取阈值
imshow("binary image", binary);
vector
vector
//轮廓发现
Mat result = src.clone();
findContours(binary, contours, hierachy, RETR_TREE, CHAIN_APPROX_SIMPLE, Point());
for (size_t t = 0; t < contours.size(); t++)
{
double area = contourArea(contours[t]);
if (area > 100000 || area < 100) //如果面积大于100000或小于100，则过滤掉该面积轮廓
continue;
drawContours(result, contours, t, Scalar(0, 0, 255), 2, 8);
double theta = calcPCAorientation(result, contours[t]);
}
imshow("contours image", result);
waitKey(0);
return 0;
}
double calcPCAorientation(Mat & image, vector
{
int size = static_cast
Mat data_pts = Mat(size, 2, CV_64FC1);//定义一个n行2列的矩阵
for (int i = 0; i < size; i++)
{
data_pts.at
data_pts.at
}
//PCA分析处理
PCA pca_analysis(data_pts, Mat(), PCA::DATA_AS_ROW);
Point cnt = Point(static_cast
static_cast
circle(image, cnt, 2, Scalar(0, 255, 0), 2, 8, 0);
vector
vector
for (int i = 0; i < 2; i++)
{
eigen_vals[i] = pca_analysis.eigenvalues.at
eigen_vectors[i] = Point2d(pca_analysis.eigenvectors.at
pca_analysis.eigenvectors.at
cout << "第" << i << "个特征值为：" << eigen_vals[i] << endl;
}
Point p1 = cnt + 0.02*Point(static_cast
Point p2 = cnt - 0.05*Point(static_cast
double angle = atan2(eigen_vectors[0].y, eigen_vectors[0].x); //反三角函数求角度
cout << "偏转角度=" << angle * (180 / CV_PI) << endl;
cout << "-------------------------------" << endl;
cout << "-------------------------------" << endl;
line(image, cnt, p1, Scalar(255, 0, 0), 2, 8, 0);
line(image, cnt, p2, Scalar(255, 125, 0), 2, 8, 0);
/*
cout << "cnt=" << cnt << endl;
cout << "p1=" << p1 << endl;
cout << "p2=" << p2 << endl;
cout << "-----------------------" << endl;
*/
return 0;
}
(2) 教学视频上PCA原理与应用代码：
#include
#include
using namespace cv;
using namespace std;
double calcPCAOrientation(vector
int main(int argc, char** argv)
{
Mat src = imread("D:/opencv3.4.9/opencv/sources/samples/data/pca_test1.jpg");
if (src.empty())
{
printf("could not load image...\n");
return -1;
}
namedWindow("input image", CV_WINDOW_AUTOSIZE);
imshow("input image", src);
Mat gray, binary;
cvtColor(src, gray, COLOR_BGR2GRAY);
threshold(gray, binary, 0, 255, THRESH_BINARY | THRESH_OTSU);
imshow("binary image", binary);
vector
vector
findContours(binary, contours, hireachy, RETR_LIST, CHAIN_APPROX_NONE);
Mat result = src.clone();
for (int i = 0; i < contours.size(); i++)
{
double area = contourArea(contours[i]);
if (area > 1e5 || area < 1e2) continue;
drawContours(result, contours, i, Scalar(0, 0, 255), 2, 8);
double theta = calcPCAOrientation(contours[i], result);
}
imshow("contours result", result);
waitKey(0);
return 0;
}
double calcPCAOrientation(vector
{
int size = static_cast
Mat data_pts =Mat(size, 2, CV_64FC1);
for (int i = 0; i < size; i++)
{
data_pts.at
data_pts.at
}
//perfrom PCA projection
PCA pca_analysis(data_pts, Mat(), CV_PCA_DATA_AS_ROW);
Point cnt = Point(static_cast
static_cast
circle(image, cnt, 2, Scalar(0, 255, 0), 2, 8, 0);
vector
vector
for (int i = 0; i < 2; i++)
{
vals[i] = pca_analysis.eigenvalues.at
printf("Th %d eigen value :%.2f\n", i, vals[i]);
vecs[i] = Point2d((pca_analysis.eigenvectors.at
pca_analysis.eigenvectors.at
}
Point p1 = cnt + 0.02*Point(static_cast
Point p2 = cnt - 0.05*Point(static_cast
line(image, cnt, p1, Scalar(255, 0, 0), 2, 8, 0);
line(image, cnt, p2, Scalar(255, 255, 255), 2, 8, 0);
double angle = atan2(vecs[0].y, vecs[0].x);
printf("angle :%.2f\n", 180 * (angle / CV_PI));
return angle;
}
P139-P140 人脸识别算法之EigenFace（注意，如果EigenFaceRecognizer不可用，则使用的连接器未添加opencv_face349d.lib）
Ptr
model->train(images, labels);
（1）CSDN 上的人脸识别算法之EigenFace代码：成功*********
#include
#include
#include
using namespace cv;
using namespace cv::face;
using namespace std;
int main(int argc, char** argv) {
string filename = string("D:/OpenCV/orl_faces/image.txt");
ifstream file(filename.c_str(), ifstream::in);
if (!file) {
printf("could not load file correctly...\n");
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';
while (getline(file, line)) {
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty()) {
//printf("path : %s\n", path.c_str());
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
if (images.size() < 1 || labels.size() < 1) {
printf("invalid image path...\n");
return -1;
}
int height = images[0].rows;
int width = images[0].cols;
printf("height : %d, width : %d\n", height, width);
Mat testSample = images[images.size() - 1];
int testLabel = labels[labels.size() - 1];
images.pop_back();
labels.pop_back();
// train it
Ptr
model->train(images, labels);
// recognition face
int predictedLabel = model->predict(testSample);
printf("actual label : %d, predict label : %d\n", testLabel, predictedLabel);
Mat eigenvalues = model->getEigenValues();
Mat W = model->getEigenVectors();
Mat mean = model->getMean();
Mat meanFace = mean.reshape(1, height);
Mat dst;
if (meanFace.channels() == 1) {
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (meanFace.channels() == 3) {
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8UC3);
}
imshow("Mean Face", dst);
// show eigen faces
for (int i = 0; i < min(10, W.cols); i++) {
Mat ev = W.col(i).clone();
Mat grayscale;
Mat eigenFace = ev.reshape(1, height);
if (eigenFace.channels() == 1) {
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (eigenFace.channels() == 3) {
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC3);
}
Mat colorface;
applyColorMap(grayscale, colorface, COLORMAP_JET);
char winTitle[128];
sprintf_s(winTitle, "eigenface_%d", i);
imshow(winTitle, colorface);
}
// 重建人脸
for (int num = min(10, W.cols); num < min(W.cols, 300); num += 15) {
Mat evs = Mat(W, Range::all(), Range(0, num));
Mat projection = LDA::subspaceProject(evs, mean, images[0].reshape(1, 1));
Mat reconstruction = LDA::subspaceReconstruct(evs, mean, projection);
Mat result = reconstruction.reshape(1, height);
if (result.channels() == 1) {
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (result.channels() == 3) {
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8UC3);
}
char winTitle[128];
sprintf_s(winTitle, "recon_face_%d", num);
imshow(winTitle, reconstruction);
}
waitKey(0);
return 0;
}
(2) 、教学视频上的人脸识别算法之EigenFace代码：演示成功********
准备：将所有图片路径整理到一个文件中代码：（最后没用上，算是一个小知识点积累）
#include
using namespace std;
int main()
{
ofstream file;
file.open("path.txt");//新建并打开文件
char str[50] = {};
for (int i = 1; i <= 40; i++) {
for (int j = 1; j <= 10; j++) {
sprintf_s(str, "orl_faces/s%d/%d.pgm;%d", i, j, i);//将数字转换成字符
file << str << endl;//写入
}
}
return 0;
}
总代码如下：
#include
#include
#include
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
using namespace cv;
using namespace cv::face;
using namespace std;
int main(int argc, char** agrv)
{
string filename = string("D:/OpenCV/orl_faces/image.txt");
ifstream file(filename.c_str(), ifstream::in);
if (!file)
{
printf("could not load file correctly...\n");
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';
while (getline(file, line))
{
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty())
{
//printf("path:%s\n", path.c_str());
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
if (images.size() < 1 || labels.size() < 1)
{
printf("invalid image path...\n");
return -1;
}
int height = images[0].rows;
int width = images[0].cols;
printf("height:%d,width :%d\n", height, width);
Mat testSample = images[images.size() - 1];//假设将最后一个照片作为要识别的对象
int testLabel = labels[labels.size() - 1];
images.pop_back();
labels.pop_back();
//train it
Ptr
model->train(images, labels);
//recognition face
int predictedLabel = model->predict(testSample);
printf("catual label :%d, predict label :%d\n", testLabel, predictedLabel);
Mat eigenvalues = model->getEigenValues();//特征值
Mat W = model->getEigenVectors();//特征向量矩阵
Mat mean = model->getMean();//平均脸
Mat meanFace = mean.reshape(1, height);
Mat dst;
if (meanFace.channels() == 1)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (meanFace.channels() == 3)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC3);
}
imshow("Mean Face", dst);
for (int i = 0; i < min(10, W.cols); i++)
{
Mat ev = W.col(i).clone();
Mat grayscale;
Mat eigenFace = ev.reshape(1, height);
if (eigenFace.channels() == 1)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (eigenFace.channels() == 3)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC3);
}
Mat colorface;
applyColorMap(grayscale, colorface, COLORMAP_BONE);
char winTitle[128];
sprintf_s(winTitle, "eigenface_%d", i);
imshow(winTitle, colorface);
}
//重建人脸
for (int num = min(10, W.cols); num < min(W.cols, 300); num += 15)
{
Mat evs = Mat(W, Range::all(), Range(0,num));
Mat projection = LDA::subspaceProject(evs, mean, images[0].reshape(1, 1));
Mat reconstruction = LDA::subspaceReconstruct(evs, mean, projection);
Mat result = reconstruction.reshape(1, height);
if (result.channels() == 1)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (result.channels() == 3)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC3);
}
char winTile[128];
sprintf_s(winTile, "recon_face_%d", num);
imshow(winTile, reconstruction);
}
waitKey(0);
return 0;
}
P141 FisherFace算法人脸识别
教学视频上的FisherFace算法人脸识别代码：
#include
#include
#include
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
using namespace cv;
using namespace cv::face;
using namespace std;
int main(int argc, char** agrv)
{
string filename = string("D:/OpenCV/orl_faces/image.txt");
ifstream file(filename.c_str(), ifstream::in);
if (!file)
{
printf("could not load file correctly...\n");
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';
while (getline(file, line))
{
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty())
{
//printf("path:%s\n", path.c_str());
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
if (images.size() < 1 || labels.size() < 1)
{
printf("invalid image path...\n");
return -1;
}
int height = images[0].rows;
int width = images[0].cols;
printf("height:%d,width :%d\n", height, width);
Mat testSample = images[images.size() - 1];//假设将最后一个照片作为要识别的对象
int testLabel = labels[labels.size() - 1];
images.pop_back();
labels.pop_back();
//train it
Ptr
model->train(images, labels);
//recognition face
int predictedLabel = model->predict(testSample);
printf("catual label :%d, predict label :%d\n", testLabel, predictedLabel);
Mat eigenvalues = model->getEigenValues();//特征值
Mat W = model->getEigenVectors();//特征向量矩阵
Mat mean = model->getMean();//平均脸
Mat meanFace = mean.reshape(1, height);
Mat dst;
if (meanFace.channels() == 1)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (meanFace.channels() == 3)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC3);
}
imshow("Mean Face", dst);
// shouw eigen faces
for (int i = 0; i < min(16, W.cols); i++)
{
Mat ev = W.col(i).clone();
Mat grayscale;
Mat eigenFace = ev.reshape(1, height);
if (eigenFace.channels() == 1)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (eigenFace.channels() == 3)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC3);
}
Mat colorface;
applyColorMap(grayscale, colorface, COLORMAP_BONE);
char winTitle[128];
sprintf_s(winTitle, "eigenface_%d", i);
imshow(winTitle, colorface);
}
//重建人脸
for (int num =0; num < min(W.cols, 16); num ++)
{
Mat evs = W.col(num);
Mat projection = LDA::subspaceProject(evs, mean, images[0].reshape(1, 1));
Mat reconstruction = LDA::subspaceReconstruct(evs, mean, projection);
Mat result = reconstruction.reshape(1, height);
if (result.channels() == 1)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (result.channels() == 3)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC3);
}
char winTile[128];
sprintf_s(winTile, "recon_face_%d", num);
imshow(winTile, reconstruction);
}
waitKey(0);
return 0;
}
P142 LBPH算法人脸识别（整数计算）
CSDN上的LBPH算法人脸识别代码：
#include
#include
#include
using namespace std;
using namespace cv;
using namespace cv::face;
//OpenCV人脸识别https://www.cnblogs.com/guoming0000/archive/2012/09/27/2706019.html
int main(int argc, char** argv)
{
String filename = string("D:/OpenCV/orl_faces/image.txt");//标签Label_Text文件
//从硬盘到内存
ifstream file(filename.c_str(), ifstream::in); // 函数可用可不用，无需返回一个标准C类型的字符串
if (!file)//打开标签文件失败
{
cout << "could not load label_file" << endl;
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';//分号
while (getline(file, line)) //getline(cin,inputLine)//cin 是正在读取的输入流,而 inputLine 是接收输入字符串的 string 变量的名称
{
stringstream liness(line);//这里采用stringstream主要作用是做字符串的分割
getline(liness, path, separator);//遇到分号就结束
getline(liness, classlabel);//继续从分号后面开始,遇到换行结束
if (!path.empty() && !classlabel.empty())
{
images.push_back(imread(path, 0));//imread(path,0)//0不能去
labels.push_back(atoi(classlabel.c_str()));//atoi字符串转换成整型数
}
}
if (images.size() <= 1 || labels.size() <= 1)//如果没有读到足够多的图片
{
cout << "读取图片数据失败..." << endl;
}
int height = images[0].rows; //得到第一张图片的高度，在下面对图像变形得到他们原始大小时需要
int width = images[0].cols;
cout << "读取的图片高度为:" << height << endl << "读取的图片宽度为:" << width << endl;
//从数据集 中移除最后一张图片,用于做测试,需要根据自己的需要进行修改
Mat testSample = images[images.size() - 2];//获取最后一张照片
int testLabel = labels[labels.size() - 2];//获取最后一个标签
images.pop_back();//移除最后一张图片
labels.pop_back();//移除最后一个标签
//lbphfaces
Ptr
model->train(images, labels);
//对测试图像进行预测,predictedLabel是预测标签结果
int predictedLabel = model->predict(testSample);
cout << "actual label: " << testLabel << endl;
cout << "predict label: " << predictedLabel << endl;
//输出LBPH算法参数
cout << "radius: " << model->getRadius() << endl;//中心像素点到周围像素点的距离
cout << "neighb: " << model->getNeighbors() << endl;//周围像素点的个数
cout << "grid_x: " << model->getGridX() << endl;//将一张图片在x方向分成几块
cout << "grid_y: " << model->getGridY() << endl;//将一张图片在y方向分成几块
cout << "thresh: " << model->getThreshold() << endl;//相似度阈值
//样本的直方图的长度
vector
cout << "Size of the histograms: " << histograms[0].total() << endl;
waitKey(0);
return 0;
}
P143-P144实时人脸识别应用开发案例******实际项目可考虑
实时人脸识别应用开发案例教学视频代码：
A：步骤1：采集人脸数据代码：
#include
#include
using namespace cv;
using namespace std;
string haar_face_datapath = "D:/Opencv3.4.6_build/install/etc/haarcascades/haarcascade_frontalface_alt_tree.xml";
int main(int agrc, char** agrv)
{
VideoCapture capture(0);//open camers
if (!capture.isOpened())
{
printf("could not open camera...\n");
return -1;
}
Size S = Size((int)capture.get(CV_CAP_PROP_FRAME_WIDTH), (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT));
int fps = capture.get(CV_CAP_PROP_FPS);
CascadeClassifier faceDetector;
faceDetector.load(haar_face_datapath);
Mat frame;
namedWindow("camera-demo", CV_WINDOW_AUTOSIZE);
vector
int count = 0;//采集人脸数据变量
while (capture.read(frame))
{
flip(frame, frame, 1);
faceDetector.detectMultiScale(frame, faces, 1.01, 1, 0, Size(100, 120), Size(400, 400));
for (int i = 0; i < faces.size(); i++)
{
if (count % 10 == 0)//大约每隔3-5秒可以采集一次
{
Mat dst;
resize(frame(faces[i]), dst, Size(100, 100));//设置好采集的图片大小尺寸
imwrite(format("D:/OpenCV/YuheFan/face_%d.jpg", count),dst);//保存采集到的照片
}
rectangle(frame, faces[i], Scalar(0, 0, 255), 2, 8, 0);
}
imshow("camera-demo", frame);
char c = waitKey(50);
if (c == 27)
{
break;
}
count++;
}
capture.release();
waitKey(0);
return 0;
}
B.步骤2，将步骤1生成的图片地址共同存放于同一个.txt文件中
C.步骤3，人脸识别算法重建、训练（算法可用以上三种算法）代码：
#include
#include
#include
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
using namespace cv;
using namespace cv::face;
using namespace std;
int main(int argc, char** agrv)
{
string filename = string("D:/OpenCV/YuheFan/image.txt");
ifstream file(filename.c_str(), ifstream::in);
if (!file)
{
printf("could not load file correctly...\n");
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';
while (getline(file, line))
{
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty())
{
//printf("path:%s\n", path.c_str());
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
if (images.size() < 1 || labels.size() < 1)
{
printf("invalid image path...\n");
return -1;
}
int height = images[0].rows;
int width = images[0].cols;
printf("height:%d,width :%d\n", height, width);
Mat testSample = images[images.size() - 1];//假设将最后一个照片作为要识别的对象
int testLabel = labels[labels.size() - 1];
images.pop_back();
labels.pop_back();
//train it
Ptr
model->train(images, labels);
//recognition face
int predictedLabel = model->predict(testSample);
printf("catual label :%d, predict label :%d\n", testLabel, predictedLabel);
Mat eigenvalues = model->getEigenValues();//特征值
Mat W = model->getEigenVectors();//特征向量矩阵
Mat mean = model->getMean();//平均脸
Mat meanFace = mean.reshape(1, height);
Mat dst;
if (meanFace.channels() == 1)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (meanFace.channels() == 3)
{
normalize(meanFace, dst, 0, 255, NORM_MINMAX, CV_8SC3);
}
imshow("Mean Face", dst);
for (int i = 0; i < min(10, W.cols); i++)
{
Mat ev = W.col(i).clone();
Mat grayscale;
Mat eigenFace = ev.reshape(1, height);
if (eigenFace.channels() == 1)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC1);
}
else if (eigenFace.channels() == 3)
{
normalize(eigenFace, grayscale, 0, 255, NORM_MINMAX, CV_8UC3);
}
Mat colorface;
applyColorMap(grayscale, colorface, COLORMAP_BONE);
char winTitle[128];
sprintf_s(winTitle, "eigenface_%d", i);
imshow(winTitle, colorface);
}
//重建人脸
for (int num = min(10, W.cols); num < min(W.cols, 300); num += 15)
{
Mat evs = Mat(W, Range::all(), Range(0, num));
Mat projection = LDA::subspaceProject(evs, mean, images[0].reshape(1, 1));
Mat reconstruction = LDA::subspaceReconstruct(evs, mean, projection);
Mat result = reconstruction.reshape(1, height);
if (result.channels() == 1)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC1);
}
else if (result.channels() == 3)
{
normalize(result, reconstruction, 0, 255, NORM_MINMAX, CV_8SC3);
}
char winTile[128];
sprintf_s(winTile, "recon_face_%d", num);
imshow(winTile, reconstruction);
}
waitKey(0);
return 0;
}
D：步骤4，打开摄像头识别代码：
#include
#include
#include
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
using namespace cv;
using namespace cv::face;
using namespace std;
string haar_face_datapath = "D:/Opencv3.4.6_build/install/etc/haarcascades/haarcascade_frontalface_alt_tree.xml";//人脸检测库
int main(int argc, char** agrv)
{
string filename = string("D:/OpenCV/YuheFan/image.txt");//由步骤2采集到的图片
ifstream file(filename.c_str(), ifstream::in);
if (!file)
{
printf("could not load file correctly...\n");
return -1;
}
string line, path, classlabel;
vector
vector
char separator = ';';
while (getline(file, line))
{
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty())
{
//printf("path:%s\n", path.c_str());
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
if (images.size() < 1 || labels.size() < 1)
{
printf("invalid image path...\n");
return -1;
}
int height = images[0].rows;
int width = images[0].cols;
printf("height:%d,width :%d\n", height, width);
Mat testSample = images[images.size() - 1];//假设将最后一个照片作为要识别的对象
int testLabel = labels[labels.size() - 1];
images.pop_back();
labels.pop_back();
//train it--训练算法采用的是EigenFaceRecognizer，也可以采用另外两个算法训练
Ptr
model->train(images, labels);
//recognition face
int predictedLabel = model->predict(testSample);
printf("catual label :%d, predict label :%d\n", testLabel, predictedLabel);
//人脸检测，为人脸识别奠定基础
CascadeClassifier faceDetector;
faceDetector.load(haar_face_datapath);
VideoCapture capture(0);//打开摄像头，开始识别喽
if (!capture.isOpened())
{
printf("could not open camera...\n");
return -1;
}
Mat frame;
namedWindow("face-recognition", CV_WINDOW_AUTOSIZE);
vector
Mat dst;
while (capture.read(frame))
{
flip(frame, frame, 1);
faceDetector.detectMultiScale(frame, faces, 1.1, 1, 0, Size(80, 100), Size(380, 400));//设置数字1越大，误识别率越低
for (int i = 0; i < faces.size(); i++)
{
Mat roi = frame(faces[i]);
cvtColor(roi, dst, COLOR_BGR2GRAY);
resize(dst, testSample, testSample.size());
int label = model->predict(testSample);
rectangle(frame, faces[i], Scalar(255, 0, 0), 2, 8, 0);
putText(frame, format("%s", (label == 1 ? "YuheFan" : "Unknow")), faces[i].tl(), FONT_HERSHEY_PLAIN, 1.0, Scalar(0, 0, 255), 2, 8);
}//putText函数参数中的label==数值，数值表示步骤2文件分号后面的数值
imshow("face-recognition", frame);
char c = waitKey(10);
if (c == 27)//按ESC键取消摄像头读取
{
break;
}
}
waitKey(0);
return 0;
}
YOLOV4代码：
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
constexpr float CONFIDENCE_THRESHOLD = 0;
constexpr float NMS_THRESHOLD = 0.4;
constexpr int NUM_CLASSES = 80;
// colors for bounding boxes
const cv::Scalar colors[] = {
{0, 255, 255},
{255, 255, 0},
{0, 255, 0},
{255, 0, 0}
};
const auto NUM_COLORS = sizeof(colors) / sizeof(colors[0]);
int main()
{
std::vector
{
std::ifstream class_file("D:/OpenCV神经网络2/yolo/coco.names");
if (!class_file)
{
std::cerr << "failed to open classes.txt\n";
return 0;
}
std::string line;
while (std::getline(class_file, line))
class_names.push_back(line);
}
cv::VideoCapture source(0);
auto net = cv::dnn::readNetFromDarknet("D:/OpenCV神经网络2/yolo/yolov4.cfg", "D:/OpenCV神经网络2/yolo/yolov4.weights");
//net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
//net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA);
net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
auto output_names = net.getUnconnectedOutLayersNames();
cv::Mat frame, blob;
std::vector
while (cv::waitKey(1) < 1)
{
source >> frame;
if (frame.empty())
{
cv::waitKey();
break;
}
auto total_start = std::chrono::steady_clock::now();
cv::dnn::blobFromImage(frame, blob, 0.00392, cv::Size(416, 416), cv::Scalar(), true, false, CV_32F);
net.setInput(blob);
auto dnn_start = std::chrono::steady_clock::now();
net.forward(detections, output_names);
auto dnn_end = std::chrono::steady_clock::now();
std::vector
std::vector
std::vector
for (auto& output : detections)
{
const auto num_boxes = output.rows;
for (int i = 0; i < num_boxes; i++)
{
auto x = output.at
auto y = output.at
auto width = output.at
auto height = output.at
cv::Rect rect(x - width / 2, y - height / 2, width, height);
for (int c = 0; c < NUM_CLASSES; c++)
{
auto confidence = *output.ptr
if (confidence >= CONFIDENCE_THRESHOLD)
{
boxes[c].push_back(rect);
scores[c].push_back(confidence);
}
}
}
}
for (int c = 0; c < NUM_CLASSES; c++)
cv::dnn::NMSBoxes(boxes[c], scores[c], 0.0, NMS_THRESHOLD, indices[c]);
for (int c = 0; c < NUM_CLASSES; c++)
{
for (size_t i = 0; i < indices[c].size(); ++i)
{
const auto color = colors[c % NUM_COLORS];
auto idx = indices[c][i];
const auto& rect = boxes[c][idx];
cv::rectangle(frame, cv::Point(rect.x, rect.y), cv::Point(rect.x + rect.width, rect.y + rect.height), color, 3);
std::ostringstream label_ss;
label_ss << class_names[c] << ": " << std::fixed << std::setprecision(2) << scores[c][idx];
auto label = label_ss.str();
int baseline;
auto label_bg_sz = cv::getTextSize(label.c_str(), cv::FONT_HERSHEY_COMPLEX_SMALL, 1, 1, &baseline);
cv::rectangle(frame, cv::Point(rect.x, rect.y - label_bg_sz.height - baseline - 10), cv::Point(rect.x + label_bg_sz.width, rect.y), color, cv::FILLED);
cv::putText(frame, label.c_str(), cv::Point(rect.x, rect.y - baseline - 5), cv::FONT_HERSHEY_COMPLEX_SMALL, 1, cv::Scalar(0, 0, 0));
}
}
auto total_end = std::chrono::steady_clock::now();
float inference_fps = 1000.0 / std::chrono::duration_cast
float total_fps = 1000.0 / std::chrono::duration_cast
std::ostringstream stats_ss;
stats_ss << std::fixed << std::setprecision(2);
stats_ss << "Inference FPS: " << inference_fps << ", Total FPS: " << total_fps;
auto stats = stats_ss.str();
int baseline;
auto stats_bg_sz = cv::getTextSize(stats.c_str(), cv::FONT_HERSHEY_COMPLEX_SMALL, 1, 1, &baseline);
cv::rectangle(frame, cv::Point(0, 0), cv::Point(stats_bg_sz.width, stats_bg_sz.height + 10), cv::Scalar(0, 0, 0), cv::FILLED);
cv::putText(frame, stats.c_str(), cv::Point(0, stats_bg_sz.height + 5), cv::FONT_HERSHEY_COMPLEX_SMALL, 1, cv::Scalar(255, 255, 255));
cv::namedWindow("output");
cv::imshow("output", frame);
}
return 0;
}
darknet.exe detector test cfg/coco.data cfg/yolov4.cfg weights/yolov4.weights data/dog.jpg
cd darknet-master\build\darknet\x64
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
// It makes sense only for video-Camera (not for video-File)
// To use - uncomment the following line. Optical-flow is supported only by OpenCV 3.x - 4.x
//#define TRACK_OPTFLOW
//#define GPU
// To use 3D-stereo camera ZED - uncomment the following line. ZED_SDK should be installed.
//#define ZED_STEREO
#include "yolo_v2_class.hpp" // imported functions from DLL
#ifdef OPENCV
#ifdef ZED_STEREO
#include
#if ZED_SDK_MAJOR_VERSION == 2
#define ZED_STEREO_2_COMPAT_MODE
#endif
#undef GPU // avoid conflict with sl::MEM::GPU
#ifdef ZED_STEREO_2_COMPAT_MODE
#pragma comment(lib, "sl_core64.lib")
#pragma comment(lib, "sl_input64.lib")
#endif
#pragma comment(lib, "sl_zed64.lib")
float getMedian(std::vector
size_t n = v.size() / 2;
std::nth_element(v.begin(), v.begin() + n, v.end());
return v[n];
}
std::vector
{
bool valid_measure;
int i, j;
const unsigned int R_max_global = 10;
std::vector
for (auto &cur_box : bbox_vect) {
const unsigned int obj_size = std::min(cur_box.w, cur_box.h);
const unsigned int R_max = std::min(R_max_global, obj_size / 2);
int center_i = cur_box.x + cur_box.w * 0.5f, center_j = cur_box.y + cur_box.h * 0.5f;
std::vector
for (int R = 0; R < R_max; R++) {
for (int y = -R; y <= R; y++) {
for (int x = -R; x <= R; x++) {
i = center_i + x;
j = center_j + y;
sl::float4 out(NAN, NAN, NAN, NAN);
if (i >= 0 && i < xyzrgba.cols && j >= 0 && j < xyzrgba.rows) {
cv::Vec4f &elem = xyzrgba.at
out.x = elem[0];
out.y = elem[1];
out.z = elem[2];
out.w = elem[3];
}
valid_measure = std::isfinite(out.z);
if (valid_measure)
{
x_vect.push_back(out.x);
y_vect.push_back(out.y);
z_vect.push_back(out.z);
}
}
}
}
if (x_vect.size() * y_vect.size() * z_vect.size() > 0)
{
cur_box.x_3d = getMedian(x_vect);
cur_box.y_3d = getMedian(y_vect);
cur_box.z_3d = getMedian(z_vect);
}
else {
cur_box.x_3d = NAN;
cur_box.y_3d = NAN;
cur_box.z_3d = NAN;
}
bbox3d_vect.emplace_back(cur_box);
}
return bbox3d_vect;
}
cv::Mat slMat2cvMat(sl::Mat &input) {
int cv_type = -1; // Mapping between MAT_TYPE and CV_TYPE
if (input.getDataType() ==
#ifdef ZED_STEREO_2_COMPAT_MODE
sl::MAT_TYPE_32F_C4
#else
sl::MAT_TYPE::F32_C4
#endif
) {
cv_type = CV_32FC4;
}
else cv_type = CV_8UC4; // sl::Mat used are either RGBA images or XYZ (4C) point clouds
return cv::Mat(input.getHeight(), input.getWidth(), cv_type, input.getPtr
#ifdef ZED_STEREO_2_COMPAT_MODE
sl::MEM::MEM_CPU
#else
sl::MEM::CPU
#endif
));
}
cv::Mat zed_capture_rgb(sl::Camera &zed) {
sl::Mat left;
zed.retrieveImage(left);
cv::Mat left_rgb;
cv::cvtColor(slMat2cvMat(left), left_rgb, CV_RGBA2RGB);
return left_rgb;
}
cv::Mat zed_capture_3d(sl::Camera &zed) {
sl::Mat cur_cloud;
zed.retrieveMeasure(cur_cloud,
#ifdef ZED_STEREO_2_COMPAT_MODE
sl::MEASURE_XYZ
#else
sl::MEASURE::XYZ
#endif
);
return slMat2cvMat(cur_cloud).clone();
}
static sl::Camera zed; // ZED-camera
#else // ZED_STEREO
std::vector
return bbox_vect;
}
#endif // ZED_STEREO
#include
#include
#ifndef CV_VERSION_EPOCH // OpenCV 3.x and 4.x
#include
#define OPENCV_VERSION CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR)"" CVAUX_STR(CV_VERSION_REVISION)
#ifndef USE_CMAKE_LIBS
#pragma comment(lib, "opencv_world" OPENCV_VERSION ".lib")
#ifdef TRACK_OPTFLOW
/*
#pragma comment(lib, "opencv_cudaoptflow" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_cudaimgproc" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")
*/
#endif // TRACK_OPTFLOW
#endif // USE_CMAKE_LIBS
#else // OpenCV 2.x
#define OPENCV_VERSION CVAUX_STR(CV_VERSION_EPOCH)"" CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR)
#ifndef USE_CMAKE_LIBS
#pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")
#pragma comment(lib, "opencv_video" OPENCV_VERSION ".lib")
#endif // USE_CMAKE_LIBS
#endif // CV_VERSION_EPOCH
using namespace std;
vector
{
vector
int subinit = 0;
for (int id = 0; id != s.length(); id++)
{
if (s[id] == sepeartor)
{
split_vector.push_back(s.substr(subinit, id - subinit));
subinit = id + 1;
}
}
split_vector.push_back(s.substr(subinit, s.length() - subinit));
return split_vector;
}
void draw_boxes(cv::Mat mat_img, std::vector
int current_det_fps = -1, int current_cap_fps = -1)
{
int const colors[6][3] = { { 1,0,1 },{ 0,0,1 },{ 0,1,1 },{ 0,1,0 },{ 1,1,0 },{ 1,0,0 } };
for (auto &i : result_vec) {
cv::Scalar color = obj_id_to_color(i.obj_id);
cv::rectangle(mat_img, cv::Rect(i.x, i.y, i.w, i.h), color, 2);
if (obj_names.size() > i.obj_id) {
std::string obj_name = obj_names[i.obj_id];
if (i.track_id > 0) obj_name += " - " + std::to_string(i.track_id);
cv::Size const text_size = getTextSize(obj_name, cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, 2, 0);
int max_width = (text_size.width > i.w + 2) ? text_size.width : (i.w + 2);
max_width = std::max(max_width, (int)i.w + 2);
//max_width = std::max(max_width, 283);
std::string coords_3d;
if (!std::isnan(i.z_3d)) {
std::stringstream ss;
ss << std::fixed << std::setprecision(2) << "x:" << i.x_3d << "m y:" << i.y_3d << "m z:" << i.z_3d << "m ";
coords_3d = ss.str();
cv::Size const text_size_3d = getTextSize(ss.str(), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, 1, 0);
int const max_width_3d = (text_size_3d.width > i.w + 2) ? text_size_3d.width : (i.w + 2);
if (max_width_3d > max_width) max_width = max_width_3d;
}
cv::rectangle(mat_img, cv::Point2f(std::max((int)i.x - 1, 0), std::max((int)i.y - 35, 0)),
cv::Point2f(std::min((int)i.x + max_width, mat_img.cols - 1), std::min((int)i.y, mat_img.rows - 1)),
color, CV_FILLED, 8, 0);
putText(mat_img, obj_name, cv::Point2f(i.x, i.y - 16), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, cv::Scalar(0, 0, 0), 2);
if (!coords_3d.empty()) putText(mat_img, coords_3d, cv::Point2f(i.x, i.y - 1), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, cv::Scalar(0, 0, 0), 1);
}
}
if (current_det_fps >= 0 && current_cap_fps >= 0) {
std::string fps_str = "FPS detection: " + std::to_string(current_det_fps) + " FPS capture: " + std::to_string(current_cap_fps);
putText(mat_img, fps_str, cv::Point2f(10, 20), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, cv::Scalar(50, 255, 0), 2);
}
}
#endif // OPENCV
void show_console_result(std::vector
if (frame_id >= 0) std::cout << " Frame: " << frame_id << std::endl;
for (auto &i : result_vec) {
if (obj_names.size() > i.obj_id) std::cout << obj_names[i.obj_id] << " - ";
std::cout << "obj_id = " << i.obj_id << ", x = " << i.x << ", y = " << i.y
<< ", w = " << i.w << ", h = " << i.h
<< std::setprecision(3) << ", prob = " << i.prob << std::endl;
}
}
std::vector
std::ifstream file(filename);
std::vector
if (!file.is_open()) return file_lines;
for (std::string line; getline(file, line);) file_lines.push_back(line);
std::cout << "object names loaded \n";
return file_lines;
}
template
class send_one_replaceable_object_t {
const bool sync;
std::atomic
public:
void send(T const& _obj) {
T *new_ptr = new T;
*new_ptr = _obj;
if (sync) {
while (a_ptr.load()) std::this_thread::sleep_for(std::chrono::milliseconds(3));
}
std::unique_ptr
}
T receive() {
std::unique_ptr
do {
while (!a_ptr.load()) std::this_thread::sleep_for(std::chrono::milliseconds(3));
ptr.reset(a_ptr.exchange(NULL));
} while (!ptr);
T obj = *ptr;
return obj;
}
bool is_object_present() {
return (a_ptr.load() != NULL);
}
send_one_replaceable_object_t(bool _sync) : sync(_sync), a_ptr(NULL)
{}
};
int main(int argc, char *argv[])
{
std::string names_file = "coco.names";
std::string cfg_file = "cfg/yolov4.cfg";
std::string weights_file = "yolov4.weights";
std::string filename;
if (argc > 4) { //voc.names yolo-voc.cfg yolo-voc.weights test.mp4
names_file = argv[1];
cfg_file = argv[2];
weights_file = argv[3];
filename = argv[4];
}
else if (argc > 1) filename = argv[1];
float const thresh = (argc > 5) ? std::stof(argv[5]) : 0.2;
Detector detector(cfg_file, weights_file);
auto obj_names = objects_names_from_file(names_file);
std::string out_videofile = "result.avi";
bool const save_output_videofile = false; // true - for history
bool const send_network = false; // true - for remote detection
bool const use_kalman_filter = false; // true - for stationary camera
bool detection_sync = true; // true - for video-file
#ifdef TRACK_OPTFLOW // for slow GPU
detection_sync = false;
Tracker_optflow tracker_flow;
//detector.wait_stream = true;
#endif // TRACK_OPTFLOW
while (true)
{
std::cout << "input image or video filename: ";
if (filename.size() == 0) std::cin >> filename;
if (filename.size() == 0) break;
try {
#ifdef OPENCV
preview_boxes_t large_preview(100, 150, false), small_preview(50, 50, true);
bool show_small_boxes = false;
std::string const file_ext = filename.substr(filename.find_last_of(".") + 1);
std::string const protocol = filename.substr(0, 7);
if (file_ext == "avi" || file_ext == "mp4" || file_ext == "mjpg" || file_ext == "mov" || // video file
protocol == "rtmp://" || protocol == "rtsp://" || protocol == "http://" || protocol == "https:/" || // video network stream
filename == "zed_camera" || file_ext == "svo" || filename == "web_camera") // ZED stereo camera
{
if (protocol == "rtsp://" || protocol == "http://" || protocol == "https:/" || filename == "zed_camera" || filename == "web_camera")
detection_sync = false;
cv::Mat cur_frame;
std::atomic
std::atomic
std::atomic
std::chrono::steady_clock::time_point steady_start, steady_end;
int video_fps = 25;
bool use_zed_camera = false;
track_kalman_t track_kalman;
#ifdef ZED_STEREO
sl::InitParameters init_params;
init_params.depth_minimum_distance = 0.5;
#ifdef ZED_STEREO_2_COMPAT_MODE
init_params.depth_mode = sl::DEPTH_MODE_ULTRA;
init_params.camera_resolution = sl::RESOLUTION_HD720;// sl::RESOLUTION_HD1080, sl::RESOLUTION_HD720
init_params.coordinate_units = sl::UNIT_METER;
init_params.camera_buffer_count_linux = 2;
if (file_ext == "svo") init_params.svo_input_filename.set(filename.c_str());
#else
init_params.depth_mode = sl::DEPTH_MODE::ULTRA;
init_params.camera_resolution = sl::RESOLUTION::HD720;// sl::RESOLUTION::HD1080, sl::RESOLUTION::HD720
init_params.coordinate_units = sl::UNIT::METER;
if (file_ext == "svo") init_params.input.setFromSVOFile(filename.c_str());
#endif
//init_params.sdk_cuda_ctx = (CUcontext)detector.get_cuda_context();
init_params.sdk_gpu_id = detector.cur_gpu_id;
if (filename == "zed_camera" || file_ext == "svo") {
std::cout << "ZED 3D Camera " << zed.open(init_params) << std::endl;
if (!zed.isOpened()) {
std::cout << " Error: ZED Camera should be connected to USB 3.0. And ZED_SDK should be installed. \n";
getchar();
return 0;
}
cur_frame = zed_capture_rgb(zed);
use_zed_camera = true;
}
#endif // ZED_STEREO
cv::VideoCapture cap;
if (filename == "web_camera") {
cap.open(0);
cap >> cur_frame;
}
else if (!use_zed_camera) {
cap.open(filename);
cap >> cur_frame;
}
#ifdef CV_VERSION_EPOCH // OpenCV 2.x
video_fps = cap.get(CV_CAP_PROP_FPS);
#else
video_fps = cap.get(cv::CAP_PROP_FPS);
#endif
cv::Size const frame_size = cur_frame.size();
//cv::Size const frame_size(cap.get(CV_CAP_PROP_FRAME_WIDTH), cap.get(CV_CAP_PROP_FRAME_HEIGHT));
std::cout << "\n Video size: " << frame_size << std::endl;
cv::VideoWriter output_video;
if (save_output_videofile)
#ifdef CV_VERSION_EPOCH // OpenCV 2.x
output_video.open(out_videofile, CV_FOURCC('D', 'I', 'V', 'X'), std::max(35, video_fps), frame_size, true);
#else
output_video.open(out_videofile, cv::VideoWriter::fourcc('D', 'I', 'V', 'X'), std::max(35, video_fps), frame_size, true);
#endif
struct detection_data_t {
cv::Mat cap_frame;
std::shared_ptr
std::vector
cv::Mat draw_frame;
bool new_detection;
uint64_t frame_id;
bool exit_flag;
cv::Mat zed_cloud;
std::queue
detection_data_t() : exit_flag(false), new_detection(false) {}
};
const bool sync = detection_sync; // sync data exchange
send_one_replaceable_object_t
prepare2detect(sync), detect2draw(sync), draw2show(sync), draw2write(sync), draw2net(sync);
std::thread t_cap, t_prepare, t_detect, t_post, t_draw, t_write, t_network;
// capture new video-frame
if (t_cap.joinable()) t_cap.join();
t_cap = std::thread([&]()
{
uint64_t frame_id = 0;
detection_data_t detection_data;
do {
detection_data = detection_data_t();
#ifdef ZED_STEREO
if (use_zed_camera) {
while (zed.grab() !=
#ifdef ZED_STEREO_2_COMPAT_MODE
sl::SUCCESS
#else
sl::ERROR_CODE::SUCCESS
#endif
) std::this_thread::sleep_for(std::chrono::milliseconds(2));
detection_data.cap_frame = zed_capture_rgb(zed);
detection_data.zed_cloud = zed_capture_3d(zed);
}
else
#endif // ZED_STEREO
{
cap >> detection_data.cap_frame;
}
fps_cap_counter++;
detection_data.frame_id = frame_id++;
if (detection_data.cap_frame.empty() || exit_flag) {
std::cout << " exit_flag: detection_data.cap_frame.size = " << detection_data.cap_frame.size() << std::endl;
detection_data.exit_flag = true;
detection_data.cap_frame = cv::Mat(frame_size, CV_8UC3);
}
if (!detection_sync) {
cap2draw.send(detection_data); // skip detection
}
cap2prepare.send(detection_data);
} while (!detection_data.exit_flag);
std::cout << " t_cap exit \n";
});
// pre-processing video frame (resize, convertion)
t_prepare = std::thread([&]()
{
std::shared_ptr
detection_data_t detection_data;
do {
detection_data = cap2prepare.receive();
det_image = detector.mat_to_image_resize(detection_data.cap_frame);
detection_data.det_image = det_image;
prepare2detect.send(detection_data); // detection
} while (!detection_data.exit_flag);
std::cout << " t_prepare exit \n";
});
// detection by Yolo
if (t_detect.joinable()) t_detect.join();
t_detect = std::thread([&]()
{
std::shared_ptr
detection_data_t detection_data;
do {
detection_data = prepare2detect.receive();
det_image = detection_data.det_image;
std::vector
if (det_image)
result_vec = detector.detect_resized(*det_image, frame_size.width, frame_size.height, thresh, true); // true
fps_det_counter++;
//std::this_thread::sleep_for(std::chrono::milliseconds(150));
detection_data.new_detection = true;
detection_data.result_vec = result_vec;
detect2draw.send(detection_data);
} while (!detection_data.exit_flag);
std::cout << " t_detect exit \n";
});
// draw rectangles (and track objects)
t_draw = std::thread([&]()
{
std::queue
detection_data_t detection_data;
do {
// for Video-file
if (detection_sync) {
detection_data = detect2draw.receive();
}
// for Video-camera
else
{
// get new Detection result if present
if (detect2draw.is_object_present()) {
cv::Mat old_cap_frame = detection_data.cap_frame; // use old captured frame
detection_data = detect2draw.receive();
if (!old_cap_frame.empty()) detection_data.cap_frame = old_cap_frame;
}
// get new Captured frame
else {
std::vector
detection_data = cap2draw.receive();
detection_data.result_vec = old_result_vec;
}
}
cv::Mat cap_frame = detection_data.cap_frame;
cv::Mat draw_frame = detection_data.cap_frame.clone();
std::vector
#ifdef TRACK_OPTFLOW
if (detection_data.new_detection) {
tracker_flow.update_tracking_flow(detection_data.cap_frame, detection_data.result_vec);
while (track_optflow_queue.size() > 0) {
draw_frame = track_optflow_queue.back();
result_vec = tracker_flow.tracking_flow(track_optflow_queue.front(), false);
track_optflow_queue.pop();
}
}
else {
track_optflow_queue.push(cap_frame);
result_vec = tracker_flow.tracking_flow(cap_frame, false);
}
detection_data.new_detection = true; // to correct kalman filter
#endif //TRACK_OPTFLOW
// track ID by using kalman filter
if (use_kalman_filter) {
if (detection_data.new_detection) {
result_vec = track_kalman.correct(result_vec);
}
else {
result_vec = track_kalman.predict();
}
}
// track ID by using custom function
else {
int frame_story = std::max(5, current_fps_cap.load());
result_vec = detector.tracking_id(result_vec, true, frame_story, 40);
}
if (use_zed_camera && !detection_data.zed_cloud.empty()) {
result_vec = get_3d_coordinates(result_vec, detection_data.zed_cloud);
}
//small_preview.set(draw_frame, result_vec);
//large_preview.set(draw_frame, result_vec);
draw_boxes(draw_frame, result_vec, obj_names, current_fps_det, current_fps_cap);
//show_console_result(result_vec, obj_names, detection_data.frame_id);
//large_preview.draw(draw_frame);
//small_preview.draw(draw_frame, true);
detection_data.result_vec = result_vec;
detection_data.draw_frame = draw_frame;
draw2show.send(detection_data);
if (send_network) draw2net.send(detection_data);
if (output_video.isOpened()) draw2write.send(detection_data);
} while (!detection_data.exit_flag);
std::cout << " t_draw exit \n";
});
// write frame to videofile
t_write = std::thread([&]()
{
if (output_video.isOpened()) {
detection_data_t detection_data;
cv::Mat output_frame;
do {
detection_data = draw2write.receive();
if (detection_data.draw_frame.channels() == 4) cv::cvtColor(detection_data.draw_frame, output_frame, CV_RGBA2RGB);
else output_frame = detection_data.draw_frame;
output_video << output_frame;
} while (!detection_data.exit_flag);
output_video.release();
}
std::cout << " t_write exit \n";
});
// send detection to the network
t_network = std::thread([&]()
{
if (send_network) {
detection_data_t detection_data;
do {
detection_data = draw2net.receive();
detector.send_json_http(detection_data.result_vec, obj_names, detection_data.frame_id, filename);
} while (!detection_data.exit_flag);
}
std::cout << " t_network exit \n";
});
// show detection
detection_data_t detection_data;
do {
steady_end = std::chrono::steady_clock::now();
float time_sec = std::chrono::duration
if (time_sec >= 1) {
current_fps_det = fps_det_counter.load() / time_sec;
current_fps_cap = fps_cap_counter.load() / time_sec;
steady_start = steady_end;
fps_det_counter = 0;
fps_cap_counter = 0;
}
detection_data = draw2show.receive();
cv::Mat draw_frame = detection_data.draw_frame;
//if (extrapolate_flag) {
// cv::putText(draw_frame, "extrapolate", cv::Point2f(10, 40), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.0, cv::Scalar(50, 50, 0), 2);
//}
cv::imshow("window name", draw_frame);
filename.replace(filename.end() - 4, filename.end(), "_yolov4_out.jpg");
int key = cv::waitKey(3); // 3 or 16ms
if (key == 'f') show_small_boxes = !show_small_boxes;
if (key == 'p') while (true) if (cv::waitKey(100) == 'p') break;
//if (key == 'e') extrapolate_flag = !extrapolate_flag;
if (key == 27) { exit_flag = true; }
//std::cout << " current_fps_det = " << current_fps_det << ", current_fps_cap = " << current_fps_cap << std::endl;
} while (!detection_data.exit_flag);
std::cout << " show detection exit \n";
cv::destroyWindow("window name");
// wait for all threads
if (t_cap.joinable()) t_cap.join();
if (t_prepare.joinable()) t_prepare.join();
if (t_detect.joinable()) t_detect.join();
if (t_post.joinable()) t_post.join();
if (t_draw.joinable()) t_draw.join();
if (t_write.joinable()) t_write.join();
if (t_network.joinable()) t_network.join();
break;
}
else if (file_ext == "txt") { // list of image files
std::ifstream file(filename);
if (!file.is_open()) std::cout << "File not found! \n";
else
for (std::string line; file >> line;) {
std::cout << line << std::endl;
cv::Mat mat_img = cv::imread(line);
std::vector
show_console_result(result_vec, obj_names);
//draw_boxes(mat_img, result_vec, obj_names);
//cv::imwrite("res_" + line, mat_img);
}
}
else { // image file
// to achive high performance for multiple images do these 2 lines in another thread
cv::Mat mat_img = cv::imread(filename);
auto det_image = detector.mat_to_image_resize(mat_img);
auto start = std::chrono::steady_clock::now();
std::vector
auto end = std::chrono::steady_clock::now();
std::chrono::duration
std::cout << " Time: " << spent.count() << " sec \n";
//result_vec = detector.tracking_id(result_vec); // comment it - if track_id is not required
draw_boxes(mat_img, result_vec, obj_names);
cv::imshow("window name", mat_img);
vector
string endname = filenamesplit[filenamesplit.size() - 1];
endname.replace(endname.end() - 4, endname.end(), "_yolov4_out.jpg");
std::string outputfile = "detect_result/" + endname;
imwrite(outputfile, mat_img);
show_console_result(result_vec, obj_names);
cv::waitKey(0);
}
#else // OPENCV
//std::vector
auto img = detector.load_image(filename);
std::vector
detector.free_image(img);
show_console_result(result_vec, obj_names);
#endif // OPENCV
}
catch (std::exception &e) { std::cerr << "exception: " << e.what() << "\n"; getchar(); }
catch (...) { std::cerr << "unknown exception \n"; getchar(); }
filename.clear();
}
return 0;
}