Opencv4+cuda 加速开发教程笔记
写在前面：2020年12月-2021年5月搞了一阵，发现老师期望过高，却没有明确指导方向，纯是自己瞎碰壁，搞不明白，改行了，学前端了，现在是2022年1月，今年毕业就入职
文章目录
- 1 配置环境
- 2 CUDA配置与测试
- 3 第一个程序
- 4 GpuMat对象操作：算术与位运算
- 5 图像直方图&图像几何操作
- 6 卷积操作 - 加速盒子模糊与高斯模糊
- 7 卷积操作-图像梯度与边缘提取
- 8 高斯双边加速，实时视频美颜
- 9 二值形态学操作
- 10 实时的颜色对象跟踪
- 11 CUDA特征 - 角点检测
- 12 实时视频背景分析
- 13 实时光流分析
- 14 ORB特征匹配加速
- 15 HOG行人检测
1 配置环境
2 CUDA配置与测试
#include3 第一个程序
int main(int argc, char **argv) {
Mat image_host = imread("lena.jpg");//CPU的内存对象
imshow("input", image_host);
//版本1.
//GpuMat image_device(image_host);//GPU的内存对象
//版本2.
GpuMat image_device;
image_device.upload(image_host);
GpuMat gray_device;
cuda::cvtColor(image_device, gray_device, COLOR_BGR2GRAY);//cuda加速的cvt版本
Mat gray_host;
gray_device.download(gray_host);
imshow("gray", gray_host);//注意：imshow显示不了GpuMat的对象
waitKey(0);
return 0;
}
4 GpuMat对象操作：算术与位运算
GpuMat与Mat的数据交换
-
cuda内存对象GpuMat
-
GpuMat与Mat的数据交换：通过upload，download进行
-
性能开销：频繁的GPU数据上传或下载都会耗时，最好只发生一次。
-
GpuMat对象有release方法，用完要及时释放。
简单的像素操作
int main(int argc, char **argv) {
Mat src1_host = imread("WindowsLogo.jpg");
Mat src2_host = imread("LinuxLogo.jpg");
//imshow("src1", src1_host);
//imshow("src2", src2_host);
//GPU对象
GpuMat src1_device, src2_device, dst_device;
src1_device.upload(src1_host);
src2_device.upload(src2_host);
cuda::add(src1_device, src2_device,dst_device);
//cuda::subtract(src1_device, src2_device, dst_device);
//cuda::multiply(src1_device, src2_device, dst_device);
Mat res;
dst_device.download(res);
//imshow("res", res);
//加减
//基于权重的融合
Mat src_host = imread("lena.jpg");
GpuMat src;
src.upload(src_host);
GpuMat black = GpuMat(src.size(),src.type());
cuda::addWeighted(src, 0.5, black, 0.5,127,dst_device);
cuda::bitwise_not(dst_device, dst_device);//位操作，每个位取反
dst_device.download(res);
imshow("Weighted add", res);
//颜色空间转换
GpuMat hsv, rgb, gray, YCrCb;
cuda::cvtColor(src, hsv, COLOR_BGR2HLS);
cuda::cvtColor(src, rgb, COLOR_BGR2RGB);
cuda::cvtColor(src, gray, COLOR_BGR2GRAY);
cuda::cvtColor(src, YCrCb, COLOR_BGR2YCrCb);
Mat hsv_host, rgb_host, gray_host, YCrCb_host;
hsv.download(hsv_host);
rgb.download(rgb_host);
gray.download(gray_host);
YCrCb.download(YCrCb_host);
waitKey(0);
return 0;
}
对于大量图像来说，cuda加速效果更明显
5 图像直方图&图像几何操作
int main(int argc, char **argv) {
Mat image_host = imread("lena.jpg");
imshow("input", image_host);
//直方图计算
GpuMat image(image_host);
vector<GpuMat> mv;
GpuMat hist, hsv;
cuda::split(image, mv);//分为三个通道
cuda::calcHist(mv[2],hist);//计算出来都是0-256，间隔是0-1
Mat hist_host;
hist.download(hist_host);
for (int i = 0; i < hist_host.cols; i++) {
int pv = hist_host.at<int>(0, i);
printf("total number: %d,of the pixel value: %d\n", pv, i);
}
//直方图均衡化
cuda::cvtColor(image, hsv, COLOR_BGR2HSV);
cuda::split(hsv, mv);
cuda::equalizeHist(mv[2], mv[2]);
cuda::merge(mv, hsv);
cuda::cvtColor(hsv, image, COLOR_HSV2BGR);
/*Mat result;
image.download(result);
imshow("eq-demo result",result);*/
//图像的几何计算
//resize and rotate
GpuMat dst;
cuda::resize(image, dst, Size(0, 0), 2, 2,INTER_CUBIC);
int cx = image.cols / 2;
int cy = image.rows / 2;
Mat M = getRotationMatrix2D(Point(cx, cy), 45, 1.0);
cuda::warpAffine(image, dst, M, image.size());
Mat result;
dst.download(result);
imshow("rotate result", result);
waitKey(0);
return 0;
}
注意：
对于直方图计算：
opencv的calcHist的Gpu版本每次只能计算图的一个通道（cpu版本可以计算多个通道）
直方图均衡化(灰度与彩色）
可以看出对比度变大。
图像几何变换
放大，缩小，旋转等
6 卷积操作 - 加速盒子模糊与高斯模糊
主要内容：
卷积模糊操作
- 盒子模糊
int main(int argc, char **argv) {
Mat image_host = imread("lena.jpg");
imshow("input", image_host);
GpuMat image, d_result3x3, d_result5x5, d_result9x9;
image.upload(image_host);
cuda::cvtColor(image, image, COLOR_BGR2BGRA);
//盒子模糊
//create box filter
auto filter_3x3 = cuda::createBoxFilter(image.type(), image.type(), Size(3, 3), Point(-1, -1));
auto filter_5x5 = cuda::createBoxFilter(image.type(), image.type(), Size(5, 5), Point(-1, -1));
auto filter_9x9 = cuda::createBoxFilter(image.type(), image.type(), Size(9, 9), Point(-1, -1));
//apply them
filter_3x3->apply(image, d_result3x3);
filter_5x5->apply(image, d_result5x5);
filter_9x9->apply(image, d_result9x9);
//下载数据
Mat result3, result5, result9;
d_result3x3.download(result3);
d_result5x5.download(result5);
d_result9x9.download(result9);
//显示数据
imshow("filter3x3 result", result3);
imshow("filter5x5 result", result5);
imshow("filter9x9 result", result9);
waitKey(0);
return 0;
}
- 高斯模糊
//高斯模糊
//create gaussion filter
auto filter_3x3 = cuda::createGaussianFilter(image.type(), image.type(), Size(5, 5), 5);
auto filter_5x5 = cuda::createGaussianFilter(image.type(), image.type(), Size(15, 15), 15);
auto filter_9x9 = cuda::createGaussianFilter(image.type(), image.type(), Size(25, 25), 25);
7 卷积操作-图像梯度与边缘提取
图像梯度
- Sobel算子
- Scharr算子
- 拉普拉斯算子
int main(int argc, char **argv) {
Mat image_host = imread("lena.jpg");
imshow("input", image_host);
image.upload(image_host);
cuda::cvtColor(image, image, COLOR_BGR2BGRA);
//image gradient
auto sobel_dx = cuda::createSobelFilter(image.type(), image.type(), 1, 0, 3);
auto sobel_dy = cuda::createSobelFilter(image.type(), image.type(), 0, 1, 3);
GpuMat grad_x, grad_y, grad_xy;
sobel_dx->apply(image, grad_x);
sobel_dy->apply(image, grad_y);
cuda::addWeighted(grad_x, 0.5, grad_y, 0.5, 0, grad_xy);
Mat grad_host;
grad_xy.download(grad_host);
imshow("gradient demo", grad_host);
waitKey(0);
return 0;
}
边缘提取操作
int main(int argc, char **argv) {
Mat image_host = imread("lena.jpg");
imshow("input", image_host);
GpuMat image;
image.upload(image_host);
cuda::cvtColor(image, image, COLOR_BGR2BGRA);
//边缘提取
GpuMat gray, edges;
cuda::cvtColor(image, gray, COLOR_BGRA2GRAY);
//auto edge_detector = cuda::createCannyEdgeDetector(50,150,3,true);//创建Canny边缘检测器
//edge_detector->detect(gray, edges);
auto laplacian_filter = cuda::createLaplacianFilter(gray.type(), gray.type(), 3, 1.0);//laplacian边缘检测器
laplacian_filter->apply(gray, edges);
Mat edges_host;
edges.download(edges_host);
imshow("Canny Edge Demo", edges_host);
waitKey(0);
return 0;
}
canny边缘提取效果（效果更好）
拉普拉斯
8 高斯双边加速，实时视频美颜
边缘保留滤波操作
void cpu_demo();
int main(int argc, char **argv) {
//Mat image_host = imread("lena.jpg");
//imshow("input", image_host);
//对图像
//GpuMat image(image_host);
//GpuMat dst;
//cuda::bilateralFilter(image, dst, 0, 100, 14, 4);//边缘保留滤波
//对视频
VideoCapture cap;
cap.open("test.mp4");
Mat frame;
GpuMat image;
GpuMat dst;
while (true) {
int64 start = getTickCount();
bool ret = cap.read(frame);
if (!ret) break;
image.upload(frame);
//cuda::bilateralFilter(image, dst, 0, 100, 14, 4);//边缘保留滤波
cuda::cvtColor(image, image, COLOR_BGR2BGRA);
cv::pyrMeanShiftFiltering(image, dst, 7, 50);
Mat result;
dst.download(result);
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
imshow("bi-filter-demo", result);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return 0;
}
void cpu_demo() {
VideoCapture cap;
cap.open("01.mp4");
Mat frame,result;
while (true) {
int64 start = getTickCount();
cap.read(frame);
cv::bilateralFilter(frame, result, 0, 100, 14, 4);//边缘保留滤波
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
imshow("cpu-demo", result);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return;
}
高斯双边滤波操作
- 图像
- 视频
cuda::cvtColor(image, image, COLOR_BGR2BGRA);
cv::pyrMeanShiftFiltering(image, dst,7, 50);//慢速算法,必须四通道
9 二值形态学操作
图形形态学操作
void cpu_demo();
int main(int argc, char **argv) {
Mat gray_host = imread("morph02.jpg",0);
imshow("input", gray_host);
GpuMat gray,binary;
gray.upload(gray_host);
cuda::threshold(gray, binary, 174, 255, THRESH_BINARY_INV);
//printf("threshod value : %.2f\n",value);
Mat se = cv::getStructuringElement(MORPH_RECT, Size(3, 3));
auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN, gray.type(), se);
morph_filter->apply(binary, binary);
Mat res;
binary.download(res);
imshow("binary", res);
waitKey(0);
return 0;
}
10 实时的颜色对象跟踪
基于二值图形分析的颜色对象跟踪
void morph_analysis_demo(){
//读视频
VideoCapture cap;
cap.open("video06.mp4");
Mat frame_host,binary;
GpuMat frame,hsv,mask;
vector<GpuMat>mv;//进行通道分离
vector<GpuMat> thres(4);//
while (true) {
int64 start = getTickCount();
bool ret = cap.read(frame_host);
if (!ret) break;
imshow("frame", frame);
frame.upload(frame_host);
cuda::cvtColor(frame, hsv, COLOR_BGR2HSV);
cuda::split(hsv, mv);
//replace inRange
cuda::threshold(mv[0], thres[0], 35, 255, THRESH_BINARY);
cuda::threshold(mv[0], thres[3], 77, 255, THRESH_BINARY);
cuda::threshold(mv[1], thres[1], 43, 255, THRESH_BINARY);
cuda::threshold(mv[2], thres[2], 46, 255, THRESH_BINARY);
cuda::bitwise_xor(thres[0], thres[3],thres[0]);
cuda::bitwise_and(thres[1], thres[0], mask);
cuda::bitwise_and(mask, thres[2], mask);
cuda::threshold(mask, mask, 66, 255, THRESH_BINARY);
Mat se = cv::getStructuringElement(MORPH_RECT, Size(7, 7));
auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN, mask.type(), se);
morph_filter->apply(mask, mask);
mask.download(binary);
imshow("mask", binary);
//连通组件分析
Mat labels = Mat::zeros(binary.size(), CV_32S);
Mat states, centroids;
int num_labels = connectedComponentsWithStats(binary, labels, states, centroids,8,4);
for (int i = 1; i < num_labels; i++) {
int cx = centroids.at<double>(i, 0);
int cy = centroids.at<double>(i, 1);
int x = states.at<int>(i, CC_STAT_LEFT);
int y = states.at<int>(i, CC_STAT_TOP);
int width = states.at<int>(i, CC_STAT_WIDTH);
int height = states.at<int>(i, CC_STAT_HEIGHT);
if (width < 50 || height < 50) {
continue;
}
circle(frame_host, Point(cx, cy), 2, Scalar(255, 0, 0), 2, 8, 0);
Rect rect(x, y, width, height);
rectangle(frame_host, rect, Scalar(0, 0, 255), 2, 8, 0);
}
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(frame_host, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
imshow("colorobject tracking -demo", frame_host);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
}
11 CUDA特征 - 角点检测
RNG rng(12345);
int main(int argc, char** argv)
{
Mat image_host = imread("building.png");
imshow("input", image_host);
GpuMat src, gray,corners;
Mat dst;
src.upload(image_host);
cuda::cvtColor(src, gray, COLOR_BGR2GRAY);
auto corner_detector = cuda::createGoodFeaturesToTrackDetector(gray.type(), 1000, 0.01, 5, 3);
corner_detector->detect(gray, corners);
corners.download(dst);
printf("number of corners:%d\n", corners.cols);
for (int i = 0; i < corners.cols; i++) {
int r = rng.uniform(0, 255);
int g = rng.uniform(0, 255);
int b = rng.uniform(0, 255);
Point2f pt = dst.at<Point2f>(0, i);
circle(image_host, pt, 3, Scalar(b, g, r), 2, 8, 0);
}
imshow("corner detect result:", image_host);
waitKey(0);
return 0;
}
12 实时视频背景分析
- 视频分析
- 加速背景分析法
- 加速光流法
int main(int argc, char **argv) {
VideoCapture cap;
cap.open("vtest.avi");
auto mog = cuda::createBackgroundSubtractorMOG2();
Mat frame;
GpuMat d_frame, d_fgmask,d_bging;
Mat fg_mask, bgimg, fgming;
Mat se = cv::getStructuringElement(MORPH_RECT, Size(5, 5));
while (true) {
int64 start = getTickCount();
bool ret = cap.read(frame);
if (!ret) break;
//背景分析
d_frame.upload(frame);
mog->apply(d_frame, d_fgmask);
mog->getBackgroundImage(d_bging);
//形态学操作
auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN,d_fgmask.type(), se);
morph_filter->apply(d_fgmask, d_fgmask);
//download fron GPU Mat
d_bging.download(bgimg);
d_fgmask.download(fg_mask);
//计算FPS
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
//显示
imshow("input", frame);
imshow("background", bgimg);
imshow("mask", fg_mask);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return 0;
}
void cpu_demo() {
VideoCapture cap;
cap.open("01.mp4");
Mat frame, result;
while (true) {
int64 start = getTickCount();
cap.read(frame);
cv::bilateralFilter(frame, result, 0, 100, 14, 4);//边缘保留滤波
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
imshow("cpu-demo", result);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return;
}
13 实时光流分析
//光流法背景分析
void optical_flow_demo()
{
VideoCapture cap;
cap.open("vtest.avi");
auto farn = cuda::FarnebackOpticalFlow::create();
Mat f,pf;
cap.read(pf);
GpuMat frame, gray, preFrme, preGray;
preFrme.upload(pf);
cuda::cvtColor(preFrme, preGray, COLOR_BGR2GRAY);
Mat hsv = Mat::zeros(preFrme.size(), preFrme.type());
GpuMat flow;
vector <Mat> mv;
split(hsv, mv);
GpuMat gMat, gAng;
Mat mag = Mat::zeros(hsv.size(), CV_32FC1);
Mat ang = Mat::zeros(hsv.size(), CV_32FC1);
gMat.upload(mag);
gAng.upload(ang);
namedWindow("input", WINDOW_AUTOSIZE);
namedWindow("optical flow demo", WINDOW_AUTOSIZE);
Mat fg_mask, bgimg, fgming;
Mat se = cv::getStructuringElement(MORPH_RECT, Size(5, 5));
while (true) {
int64 start = getTickCount();
bool ret = cap.read(frame);
if (!ret) break;
//光流分析
frame.upload(f);
cuda::cvtColor(frame, gray, COLOR_BGR2GRAY);
farn->calc(preGray, gray, flow);
//坐标转换
vector<GpuMat> mm;
cuda::split(flow, mm);
cuda::cartToPolar(mm[0], mm[1], gMat, gAng);
cuda::normalize(gMat, gMat, 0, 255, NORM_MINMAX, CV_32FC1);
gMat.download(mag);
gAng.download(ang);
//显示
ang = ang * 180 / CV_PI / 2.0;
convertScaleAbs(mag, mag);
convertScaleAbs(ang, mag);
mv[0] = ang;
mv[1] = Scalar(255);
mv[2] = mag;
merge(mv, hsv);
Mat bgr;
cv::cvtColor(hsv, bgr, COLOR_HSV2BGR);
//计算FPS
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
//显示
gray.copyTo(preGray);
imshow("input", f);
imshow("optical flow demo ",bgr);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return;
}
14 ORB特征匹配加速
特征提取与特征匹配
注意：opencv中的opencvDN模块可以实现，效果比cuda好。
ORB特征检测
ORB特征匹配
//关键代码如下：
// 对象检测
auto orb = cuda::ORB::create();
// Detect feature points and extract corresponding descriptors
orb->detectAndCompute(d_object_image, cuda::GpuMat(), h_keypoints_object, d_descriptors_object);
orb->detectAndCompute(d_scene_image, cuda::GpuMat(), h_keypoints_scene, d_descriptors_scene);
// Brute Force Violence Matcher
Ptr< cuda::DescriptorMatcher > matcher = cuda::DescriptorMatcher::createBFMatcher(NORM_HAMMING);
vector< vector< DMatch> > d_matches;
matcher->knnMatch(d_descriptors_object, d_descriptors_scene, d_matches, 2);
std::cout << "match size:" << d_matches.size() << endl;
std::vector< DMatch > good_matches;
for (int k = 0; k < std::min(h_keypoints_object.size() - 1, d_matches.size()); k++)
{
if ((d_matches[k][0].distance < 0.9*(d_matches[k][1].distance)) &&
((int)d_matches[k].size() <= 2 && (int)d_matches[k].size()>0))
{
good_matches.push_back(d_matches[k][0]);
}
}
std::cout << "size:" << good_matches.size() << endl;
opencv4 把sift,surf都从realease移到扩展模块（专利）
现在可以用ORB
15 HOG行人检测
级联检测器再有都是基于cnn的
对象检测HOG，行人检测的早期算法，传统算法，hog特征提取+SVM分类，结合在一起对符合的hog描述算子符进行分类，找出行人。
int main(int argc, char **argv) {
VideoCapture cap;
cap.open("vtest.avi");
Mat f;
GpuMat frame, gray;
namedWindow("input", WINDOW_AUTOSIZE);
namedWindow("people detector demo", WINDOW_AUTOSIZE);
//创建检测器
auto hog = cuda::HOG::create();
hog->setSVMDetector(hog->getDefaultPeopleDetector());
vector objects;
while (true) {
int64 start = getTickCount();
bool ret = cap.read(f);
if (!ret) break;
imshow("input", f);
//Hog decetor
frame.upload(f);
cuda::cvtColor(frame, gray, COLOR_BGR2GRAY);
hog->detectMultiScale(gray, objects);
//绘制检测
for (int i = 0; i < objects.size(); i++) {
rectangle(f, objects[i], Scalar(0, 0, 255), 2, 8, 0);
}
//计算FPS
double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
//显示
imshow("peple dector", f);
char c = waitKey(1);
if (c == 27) {
break;//如果是 esc 则break
}
}
waitKey(0);
return 0;
}