Opencv4+cuda 加速开发教程笔记

写在前面：2020年12月-2021年5月搞了一阵，发现老师期望过高，却没有明确指导方向，纯是自己瞎碰壁，搞不明白，改行了，学前端了，现在是2022年1月，今年毕业就入职

1 配置环境

2 CUDA配置与测试

---

#include
#include
#include

using namespace cv;
using namespace cv::cuda;
using namespace std;

int main(int argc, char **argv) {
	cuda::printCudaDeviceInfo(cuda::getDevice());
	int count = getCudaEnabledDeviceCount();
	printf("device count:%d\n", count);
	return 0;
}

3 第一个程序

---

int main(int argc, char **argv) {
	Mat image_host = imread("lena.jpg");//CPU的内存对象
	imshow("input", image_host);

	//版本1.
	//GpuMat image_device(image_host);//GPU的内存对象
	//版本2.
	GpuMat image_device;
	image_device.upload(image_host);
	GpuMat gray_device;
	cuda::cvtColor(image_device, gray_device, COLOR_BGR2GRAY);//cuda加速的cvt版本
	
	Mat gray_host;
	gray_device.download(gray_host); 
	imshow("gray", gray_host);//注意：imshow显示不了GpuMat的对象
	waitKey(0);
	return 0;
}

4 GpuMat对象操作：算术与位运算

---

GpuMat与Mat的数据交换

• cuda内存对象GpuMat

• GpuMat与Mat的数据交换：通过upload，download进行

• 性能开销：频繁的GPU数据上传或下载都会耗时，最好只发生一次。

• GpuMat对象有release方法，用完要及时释放。

简单的像素操作

int main(int argc, char **argv) {

	Mat src1_host = imread("WindowsLogo.jpg");
	Mat src2_host = imread("LinuxLogo.jpg");
	//imshow("src1", src1_host);
	//imshow("src2", src2_host);

	//GPU对象
	GpuMat src1_device, src2_device, dst_device;
	src1_device.upload(src1_host);
	src2_device.upload(src2_host);
	cuda::add(src1_device, src2_device,dst_device);
	//cuda::subtract(src1_device, src2_device, dst_device);
	//cuda::multiply(src1_device, src2_device, dst_device);

	Mat res;
	dst_device.download(res);
	//imshow("res", res);

	//加减
	//基于权重的融合
	Mat src_host = imread("lena.jpg");
	GpuMat src;
	src.upload(src_host);
	GpuMat black = GpuMat(src.size(),src.type());
	cuda::addWeighted(src, 0.5, black, 0.5,127,dst_device);
	cuda::bitwise_not(dst_device, dst_device);//位操作，每个位取反

	dst_device.download(res);
	imshow("Weighted add", res);

	//颜色空间转换
	GpuMat hsv, rgb, gray, YCrCb;
	cuda::cvtColor(src, hsv, COLOR_BGR2HLS);
	cuda::cvtColor(src, rgb, COLOR_BGR2RGB);
	cuda::cvtColor(src, gray, COLOR_BGR2GRAY);
	cuda::cvtColor(src, YCrCb, COLOR_BGR2YCrCb);

	Mat hsv_host, rgb_host, gray_host, YCrCb_host;
	hsv.download(hsv_host);
	rgb.download(rgb_host);
	gray.download(gray_host);
	YCrCb.download(YCrCb_host);

	waitKey(0);
	return 0;
}

对于大量图像来说，cuda加速效果更明显

5 图像直方图&图像几何操作

---

int main(int argc, char **argv) {

	
	Mat image_host = imread("lena.jpg");
	imshow("input", image_host);

	//直方图计算
	GpuMat image(image_host);
	vector<GpuMat> mv;
	GpuMat hist, hsv;
	cuda::split(image, mv);//分为三个通道
	cuda::calcHist(mv[2],hist);//计算出来都是0-256，间隔是0-1

	Mat hist_host;
	hist.download(hist_host);
	for (int i = 0; i < hist_host.cols; i++) {
		int pv = hist_host.at<int>(0, i);
		printf("total number: %d,of the pixel value: %d\n", pv, i);
	}
	
	//直方图均衡化
	cuda::cvtColor(image, hsv, COLOR_BGR2HSV);
	cuda::split(hsv, mv);
	cuda::equalizeHist(mv[2], mv[2]);
	cuda::merge(mv, hsv);
	cuda::cvtColor(hsv, image, COLOR_HSV2BGR);
	/*Mat result;
	image.download(result);
	imshow("eq-demo result",result);*/

	//图像的几何计算
	//resize and rotate
	GpuMat dst;
	cuda::resize(image, dst, Size(0, 0), 2, 2,INTER_CUBIC);
	int cx = image.cols / 2;
	int cy = image.rows / 2;
	Mat M = getRotationMatrix2D(Point(cx, cy), 45, 1.0);
	cuda::warpAffine(image, dst, M, image.size());

	Mat result;
	dst.download(result);
	imshow("rotate result", result); 


	waitKey(0);
	return 0;
}

注意：

对于直方图计算：

opencv的calcHist的Gpu版本每次只能计算图的一个通道（cpu版本可以计算多个通道）

直方图均衡化(灰度与彩色）

可以看出对比度变大。

图像几何变换

放大，缩小，旋转等

6 卷积操作 - 加速盒子模糊与高斯模糊

---

主要内容：

卷积模糊操作

• 盒子模糊

int main(int argc, char **argv) {

	Mat image_host = imread("lena.jpg");

	imshow("input", image_host);

	GpuMat image, d_result3x3, d_result5x5, d_result9x9;
	image.upload(image_host);
	cuda::cvtColor(image, image, COLOR_BGR2BGRA);

	//盒子模糊
	//create box filter
	auto filter_3x3 = cuda::createBoxFilter(image.type(), image.type(), Size(3, 3), Point(-1, -1));
	auto filter_5x5 = cuda::createBoxFilter(image.type(), image.type(), Size(5, 5), Point(-1, -1));
	auto filter_9x9 = cuda::createBoxFilter(image.type(), image.type(), Size(9, 9), Point(-1, -1));
	
	//apply them
	filter_3x3->apply(image, d_result3x3);
	filter_5x5->apply(image, d_result5x5);
	filter_9x9->apply(image, d_result9x9);

    //下载数据
	Mat result3, result5, result9;
	d_result3x3.download(result3);
	d_result5x5.download(result5);
	d_result9x9.download(result9);

	//显示数据
	imshow("filter3x3 result", result3);
	imshow("filter5x5 result", result5);
	imshow("filter9x9 result", result9);

	waitKey(0);
	return 0;
}

• 高斯模糊

	//高斯模糊
	//create gaussion filter
	auto filter_3x3 = cuda::createGaussianFilter(image.type(), image.type(), Size(5, 5), 5);
	auto filter_5x5 = cuda::createGaussianFilter(image.type(), image.type(), Size(15, 15), 15);
	auto filter_9x9 = cuda::createGaussianFilter(image.type(), image.type(), Size(25, 25), 25);
	

7 卷积操作-图像梯度与边缘提取

---

图像梯度

• Sobel算子
• Scharr算子
• 拉普拉斯算子

int main(int argc, char **argv) {

	Mat image_host = imread("lena.jpg");

	imshow("input", image_host);

	image.upload(image_host);
	cuda::cvtColor(image, image, COLOR_BGR2BGRA);

	//image gradient
	auto sobel_dx = cuda::createSobelFilter(image.type(), image.type(), 1, 0, 3);
	auto sobel_dy = cuda::createSobelFilter(image.type(), image.type(), 0, 1, 3);

	GpuMat grad_x, grad_y, grad_xy;
	sobel_dx->apply(image, grad_x);
	sobel_dy->apply(image, grad_y);
	cuda::addWeighted(grad_x, 0.5, grad_y, 0.5, 0, grad_xy);

	Mat grad_host;
	grad_xy.download(grad_host);
	imshow("gradient demo", grad_host);

	waitKey(0);
	return 0;
}

边缘提取操作

int main(int argc, char **argv) {

	Mat image_host = imread("lena.jpg");

	imshow("input", image_host);

	GpuMat image;
	image.upload(image_host);
	cuda::cvtColor(image, image, COLOR_BGR2BGRA);

	//边缘提取
	GpuMat gray, edges;
	cuda::cvtColor(image, gray, COLOR_BGRA2GRAY);
	//auto edge_detector = cuda::createCannyEdgeDetector(50,150,3,true);//创建Canny边缘检测器
	//edge_detector->detect(gray, edges);
	auto laplacian_filter = cuda::createLaplacianFilter(gray.type(), gray.type(), 3, 1.0);//laplacian边缘检测器
	laplacian_filter->apply(gray, edges);

	Mat edges_host;
	edges.download(edges_host);
	imshow("Canny Edge Demo", edges_host);

	waitKey(0);
	return 0;
}

canny边缘提取效果（效果更好）

拉普拉斯

8 高斯双边加速，实时视频美颜

---

边缘保留滤波操作

void cpu_demo();

int main(int argc, char **argv) {
	//Mat image_host = imread("lena.jpg");
	//imshow("input", image_host);

	//对图像
	//GpuMat image(image_host);
	//GpuMat dst;
	//cuda::bilateralFilter(image, dst, 0, 100, 14, 4);//边缘保留滤波

	//对视频
	VideoCapture cap;
	cap.open("test.mp4");
	Mat frame;
	GpuMat image;
	GpuMat dst;
	while (true) {
		int64 start = getTickCount();
		bool ret = cap.read(frame);
		if (!ret) break;
		image.upload(frame);
		//cuda::bilateralFilter(image, dst, 0, 100, 14, 4);//边缘保留滤波
		cuda::cvtColor(image, image, COLOR_BGR2BGRA);
		cv::pyrMeanShiftFiltering(image, dst, 7, 50);
		Mat result;
		dst.download(result); 
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
		imshow("bi-filter-demo", result);
		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return 0;
}

void cpu_demo() {
	VideoCapture cap;
	cap.open("01.mp4");
	Mat frame,result;
	while (true) {
		int64 start = getTickCount();
		cap.read(frame);
		cv::bilateralFilter(frame, result, 0, 100, 14, 4);//边缘保留滤波
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
		imshow("cpu-demo", result);
		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return;
}

高斯双边滤波操作

• 图像
• 视频

cuda::cvtColor(image, image, COLOR_BGR2BGRA);

cv::pyrMeanShiftFiltering(image, dst,7, 50);//慢速算法,必须四通道

9 二值形态学操作

---

图形形态学操作

void cpu_demo();

int main(int argc, char **argv) {
	Mat gray_host = imread("morph02.jpg",0);
	imshow("input", gray_host);


	GpuMat gray,binary;
	gray.upload(gray_host);
	cuda::threshold(gray, binary, 174, 255, THRESH_BINARY_INV);
	//printf("threshod value : %.2f\n",value);
	Mat se = cv::getStructuringElement(MORPH_RECT, Size(3, 3));
	auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN, gray.type(), se);
	morph_filter->apply(binary, binary);

	Mat res;
	binary.download(res);
	imshow("binary", res);
	waitKey(0);
	return 0;
}

10 实时的颜色对象跟踪

---

基于二值图形分析的颜色对象跟踪

void morph_analysis_demo(){
	//读视频
	VideoCapture cap;
	cap.open("video06.mp4");
	Mat frame_host,binary;
	GpuMat frame,hsv,mask;
	vector<GpuMat>mv;//进行通道分离
	vector<GpuMat> thres(4);//
	while (true) {
		int64 start = getTickCount();
		bool ret = cap.read(frame_host);
		if (!ret) break;
		imshow("frame", frame);
		frame.upload(frame_host);
		cuda::cvtColor(frame, hsv, COLOR_BGR2HSV);
		cuda::split(hsv, mv);

		//replace inRange
		cuda::threshold(mv[0], thres[0], 35, 255, THRESH_BINARY);
		cuda::threshold(mv[0], thres[3], 77, 255, THRESH_BINARY);
		cuda::threshold(mv[1], thres[1], 43, 255, THRESH_BINARY);
		cuda::threshold(mv[2], thres[2], 46, 255, THRESH_BINARY);
		cuda::bitwise_xor(thres[0], thres[3],thres[0]);

		cuda::bitwise_and(thres[1], thres[0], mask);
		cuda::bitwise_and(mask, thres[2], mask);
		cuda::threshold(mask, mask, 66, 255, THRESH_BINARY);

		Mat se = cv::getStructuringElement(MORPH_RECT, Size(7, 7));
		auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN, mask.type(), se);
		morph_filter->apply(mask, mask);

		mask.download(binary);
		imshow("mask", binary);

		//连通组件分析
		Mat labels = Mat::zeros(binary.size(), CV_32S);
		Mat states, centroids;
		int num_labels = connectedComponentsWithStats(binary, labels, states, centroids,8,4);
		for (int i = 1; i < num_labels; i++) {
			int cx = centroids.at<double>(i, 0);
			int cy = centroids.at<double>(i, 1);

			int x = states.at<int>(i, CC_STAT_LEFT);
			int y = states.at<int>(i, CC_STAT_TOP);
			int width = states.at<int>(i, CC_STAT_WIDTH);
			int height = states.at<int>(i, CC_STAT_HEIGHT);
			if (width < 50 || height < 50) {
				continue;
			}
			circle(frame_host, Point(cx, cy), 2, Scalar(255, 0, 0), 2, 8, 0);
			Rect rect(x, y, width, height);
			rectangle(frame_host, rect, Scalar(0, 0, 255), 2, 8, 0);
		}

		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(frame_host, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
		imshow("colorobject tracking -demo", frame_host);
		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}

}

11 CUDA特征 - 角点检测

RNG rng(12345);
int main(int argc, char** argv)
{
	Mat image_host = imread("building.png");
	imshow("input", image_host);

	GpuMat src, gray,corners;
	Mat dst;
	src.upload(image_host);
	cuda::cvtColor(src, gray, COLOR_BGR2GRAY);
	auto corner_detector = cuda::createGoodFeaturesToTrackDetector(gray.type(), 1000, 0.01, 5, 3);
	corner_detector->detect(gray, corners);
	corners.download(dst);
	printf("number of corners:%d\n", corners.cols);
	for (int i = 0; i < corners.cols; i++) {
		int r = rng.uniform(0, 255);
		int g = rng.uniform(0, 255);
		int b = rng.uniform(0, 255);
		Point2f pt = dst.at<Point2f>(0, i);
		circle(image_host, pt, 3, Scalar(b, g, r), 2, 8, 0);
	}

	imshow("corner detect result:", image_host);
	waitKey(0);
	return 0;
}

12 实时视频背景分析

• 视频分析
• 加速背景分析法
• 加速光流法

int main(int argc, char **argv) {

	VideoCapture cap;
	cap.open("vtest.avi");

	auto mog = cuda::createBackgroundSubtractorMOG2();
	Mat frame;
	GpuMat d_frame, d_fgmask,d_bging;
	Mat fg_mask, bgimg, fgming;
	Mat se = cv::getStructuringElement(MORPH_RECT, Size(5, 5));
	while (true) {
		int64 start = getTickCount();
		bool ret = cap.read(frame);
		if (!ret) break;
		
		//背景分析
		d_frame.upload(frame);
		mog->apply(d_frame, d_fgmask);
		mog->getBackgroundImage(d_bging);

		//形态学操作
		auto morph_filter = cuda::createMorphologyFilter(MORPH_OPEN,d_fgmask.type(), se);
		morph_filter->apply(d_fgmask, d_fgmask);

		//download fron GPU Mat
		d_bging.download(bgimg);
		d_fgmask.download(fg_mask);


		//计算FPS
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
		
		//显示
		imshow("input", frame);
		imshow("background", bgimg);
		imshow("mask", fg_mask);

		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return 0;
}

void cpu_demo() {
	VideoCapture cap;
	cap.open("01.mp4");
	Mat frame, result;
	while (true) {
		int64 start = getTickCount();
		cap.read(frame);
		cv::bilateralFilter(frame, result, 0, 100, 14, 4);//边缘保留滤波
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(result, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);
		imshow("cpu-demo", result);
		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return;
}

13 实时光流分析

//光流法背景分析
void optical_flow_demo()
{
	VideoCapture cap;
	cap.open("vtest.avi");

	auto farn = cuda::FarnebackOpticalFlow::create();
	Mat f,pf;
	cap.read(pf);

	GpuMat frame, gray, preFrme, preGray;
	preFrme.upload(pf);
	cuda::cvtColor(preFrme, preGray, COLOR_BGR2GRAY);
	Mat hsv = Mat::zeros(preFrme.size(), preFrme.type());

	GpuMat flow;
	vector <Mat> mv;
	split(hsv, mv);
	GpuMat gMat, gAng;
	Mat mag = Mat::zeros(hsv.size(), CV_32FC1);
	Mat ang = Mat::zeros(hsv.size(), CV_32FC1);

	gMat.upload(mag);
	gAng.upload(ang);

	namedWindow("input", WINDOW_AUTOSIZE);
	namedWindow("optical flow demo", WINDOW_AUTOSIZE);


	Mat fg_mask, bgimg, fgming;
	Mat se = cv::getStructuringElement(MORPH_RECT, Size(5, 5));
	while (true) {
		int64 start = getTickCount();
		bool ret = cap.read(frame);
		if (!ret) break;

		//光流分析
		frame.upload(f);
		cuda::cvtColor(frame, gray, COLOR_BGR2GRAY);
		farn->calc(preGray, gray, flow);

		//坐标转换
		vector<GpuMat> mm;
		cuda::split(flow, mm);
		cuda::cartToPolar(mm[0], mm[1], gMat, gAng);
		cuda::normalize(gMat, gMat, 0, 255, NORM_MINMAX, CV_32FC1);
		gMat.download(mag);
		gAng.download(ang);

		//显示
		ang = ang * 180 / CV_PI / 2.0;
		convertScaleAbs(mag, mag);
		convertScaleAbs(ang, mag);
		mv[0] = ang;
		mv[1] = Scalar(255);
		mv[2] = mag;
		merge(mv, hsv);
		Mat bgr;
		cv::cvtColor(hsv, bgr, COLOR_HSV2BGR);

		//计算FPS
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);

		//显示
		gray.copyTo(preGray);
		imshow("input", f);
		imshow("optical flow demo ",bgr);

		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return;
}

14 ORB特征匹配加速

---

特征提取与特征匹配

注意：opencv中的opencvDN模块可以实现，效果比cuda好。

ORB特征检测

ORB特征匹配

//关键代码如下：

// 对象检测
	auto orb = cuda::ORB::create();
	// Detect feature points and extract corresponding descriptors
	orb->detectAndCompute(d_object_image, cuda::GpuMat(), h_keypoints_object, d_descriptors_object);
	orb->detectAndCompute(d_scene_image, cuda::GpuMat(), h_keypoints_scene, d_descriptors_scene);

	// Brute Force Violence Matcher
	Ptr< cuda::DescriptorMatcher > matcher = cuda::DescriptorMatcher::createBFMatcher(NORM_HAMMING);
	vector< vector< DMatch> > d_matches;
	matcher->knnMatch(d_descriptors_object, d_descriptors_scene, d_matches, 2);

	std::cout << "match size:" << d_matches.size() << endl;
	std::vector< DMatch > good_matches;
	for (int k = 0; k < std::min(h_keypoints_object.size() - 1, d_matches.size()); k++)
	{
		if ((d_matches[k][0].distance < 0.9*(d_matches[k][1].distance)) &&
			((int)d_matches[k].size() <= 2 && (int)d_matches[k].size()>0))
		{
			good_matches.push_back(d_matches[k][0]);
		}
	}
	std::cout << "size:" << good_matches.size() << endl;

opencv4 把sift,surf都从realease移到扩展模块（专利）

现在可以用ORB

15 HOG行人检测

---

级联检测器再有都是基于cnn的

对象检测HOG，行人检测的早期算法，传统算法，hog特征提取+SVM分类，结合在一起对符合的hog描述算子符进行分类，找出行人。

int main(int argc, char **argv) {

	VideoCapture cap;
	cap.open("vtest.avi");

	Mat f;
	GpuMat frame, gray;
	namedWindow("input", WINDOW_AUTOSIZE);
	namedWindow("people detector  demo", WINDOW_AUTOSIZE);

	//创建检测器
	auto hog = cuda::HOG::create();
	hog->setSVMDetector(hog->getDefaultPeopleDetector());
	vector objects;
	while (true) {
		int64 start = getTickCount();
		bool ret = cap.read(f);
		if (!ret) break;
		imshow("input", f);

		//Hog decetor
		frame.upload(f);
		cuda::cvtColor(frame, gray, COLOR_BGR2GRAY);
		hog->detectMultiScale(gray, objects);

		//绘制检测
		for (int i = 0; i < objects.size(); i++) {
			rectangle(f, objects[i], Scalar(0, 0, 255), 2, 8, 0);
		}


		//计算FPS
		double fps = getTickFrequency() / (getTickCount() - start);//fps 每秒多少帧（正常播放速度再1s20-30帧）
		putText(frame, format("PFS:%.2f", fps), Point(50, 50), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 0, 255), 2, 8);

		//显示
		imshow("peple dector", f);


		char c = waitKey(1);
		if (c == 27) {
			break;//如果是 esc 则break
		}
	}
	waitKey(0);
	return 0;
}