Opencv C++ 调用 tensorflow 模型、caffe模型

C++ Opencv 部署模型

调用 TensorFlow 模型

重要参考链接：

• C++调用tensorflow训练好的SSD物体检测模型-opencv3.4.3
• OpenCV调用TensorFlow预训练模型

1. 使用VS2015新建 空项目。

2. 配置项目的包含目录，库目录和附加依赖项
   

3. 新建源文件 main.cpp，内容如下：

   #include
   #include
   #include 
   #include
   #include
   #include
   
   using namespace std;
   using namespace cv;
   
   
   const char* classNames[]= {"background", "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
   "fire hydrant", "background", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "background", "backpack",
   "umbrella", "background", "background", "handbag", "tie", "suitcase", "frisbee","skis", "snowboard", "sports ball", "kite", "baseball bat","baseball glove", "skateboard", "surfboard", "tennis racket",
   "bottle", "background", "wine glass", "cup", "fork", "knife", "spoon","bowl", "banana",  "apple", "sandwich", "orange","broccoli", "carrot", "hot dog",  "pizza", "donut",
   "cake", "chair", "couch", "potted plant", "bed", "background", "dining table", "background", "background", "toilet", "background","tv", "laptop", "mouse", "remote", "keyboard",
   "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "background","book", "clock", "vase", "scissors","teddy bear", "hair drier", "toothbrush"};
   
   int main() 
   {
   	String weights = "models/frozen_inference_graph.pb";
   	String prototxt = "models/ssd_mobilenet_v1_coco.pbtxt";
   	const size_t width = 300;
   	const size_t height = 300;
   
   	VideoCapture capture;
   	capture.open(0);
   	namedWindow("input", CV_WINDOW_AUTOSIZE);
   	int w = capture.get(CAP_PROP_FRAME_WIDTH);
   	int h = capture.get(CAP_PROP_FRAME_HEIGHT);
   
   	printf("frame width : %d, frame height : %d", w, h);
   
   	// set up net
   	dnn::Net net = cv::dnn::readNetFromTensorflow(weights, prototxt);
   
   	Mat frame;
   	/*
   	while (1)	// 模式1：测试单张图像
   	{
   		frame = imread("models/car.jpg");
   		imshow("input", frame);
   	*/
   
   	while (capture.read(frame))		// 模式2：调用摄像头
   	{
   
   		//预测
   		cv::Mat inputblob = cv::dnn::blobFromImage(frame, 1. / 255, Size(width, height));
   		net.setInput(inputblob);
   		Mat output = net.forward();
   
   		//检测
   		Mat detectionMat(output.size[2], output.size[3], CV_32F, output.ptr<float>());
   		float confidence_threshold = 0.5;
   		for (int i = 0; i < detectionMat.rows; i++) {
   			float confidence = detectionMat.at<float>(i, 2);
   			if (confidence > confidence_threshold) {
   				size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
   				float tl_x = detectionMat.at<float>(i, 3) * frame.cols;
   				float tl_y = detectionMat.at<float>(i, 4) * frame.rows;
   				float br_x = detectionMat.at<float>(i, 5) * frame.cols;
   				float br_y = detectionMat.at<float>(i, 6) * frame.rows;
   
   				Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
   				rectangle(frame, object_box, Scalar(0, 255, 0), 2, 8, 0);
   				putText(frame, format("%s", classNames[objIndex]), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(0, 255, 0), 2);
   			}
   		}
   		imshow("ssd-video-demo", frame);
   		char c = waitKey(5);
   		if (c == 27) 
   		{ // ESC退出
   			break;
   		}
   	}
   	capture.release();
   	waitKey(0);
   	return 0;
   }
   
   

4. 在该项目下新建models文件夹，放入测试图像和模型。

   • 图像可以上网随意下载。
   • pb文件：http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_11_06_2017.tar.gz 中的 frozen_inference_graph.pb 文件
   • pbtxt文件：https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/ssd_mobilenet_v1_coco.pbtxt
   • 问题：Unknown enumeration value of “DT_RESOURCE” for field “type”。
     答：pbtxt文件错误，使用ssd_mobilenet_v1_coco_11_06_2017.tar.gz中的默认pbtxt文件会造成此问题，更换为上述opencv的pbtxt则没问题。或者将tensorflow的pb文件转化为pbtxt，这个方案没有尝试。

5. 运行程序main.cpp，输出结果

调用 Caffe 分类模型（bvlc_googlenet.caffemodel）

重要参考链接：

• opencv3.4.3的DNN模块调用bvlc_googlenet.caffemodel实现图像分类
• 使用训练好的googlenet caffemodel进行图片分类

1. 使用VS2015新建 空项目。

2. 配置项目的包含目录，库目录和附加依赖项
   

3. 新建源文件 main.cpp，内容如下：

   //use opencv_dnn module for image classification by using GoogLeNet trained network
   #include 
   #include 
   #include 
   using namespace cv;
   using namespace cv::dnn;
   using namespace std;
   String modelTxt = "models/bvlc_googlenet.prototxt";
   String modelBin = "models/bvlc_googlenet.caffemodel";
   String labelFile = "models/synset_words.txt";
   vector<String> readClasslabels();
   int main(int argc, char** argv) {
   	Mat testImage = imread("models/car.jpg");
   	if (testImage.empty()) {
   		printf("could not load image...\n");
   		return -1;
   	}
   	// create googlenet with caffemodel text and bin
   	Net net = dnn::readNetFromCaffe(modelTxt, modelBin);
   	if (net.empty())
   	{
   		std::cerr << "Can't load network by using the following files: " << std::endl;
   		std::cerr << "prototxt:   " << modelTxt << std::endl;
   		std::cerr << "caffemodel: " << modelBin << std::endl;
   		return -1;
   	}
   	// 读取分类数据
   	vector<String> labels = readClasslabels();
   	//GoogLeNet accepts only 224x224 RGB-images
   	Mat inputBlob = blobFromImage(testImage, 1, Size(224, 224), Scalar(104, 117, 123));//mean: Scalar(104, 117, 123)
   	// 支持1000个图像分类检测
   	Mat prob;
   	// 循环10+
   	for (int i = 0; i < 10; i++)
   	{
   		// 输入
   		net.setInput(inputBlob, "data");
   		// 分类预测
   		prob = net.forward("prob");
   	}
   	// 读取分类索引，最大与最小值
   	Mat probMat = prob.reshape(1, 1); //reshape the blob to 1x1000 matrix // 1000个分类
   	Point classNumber;
   	double classProb;
   	minMaxLoc(probMat, NULL, &classProb, NULL, &classNumber); // 可能性最大的一个
   	int classIdx = classNumber.x; // 分类索引号
   	printf("\n current image classification : %s, possible : %.2f \n", labels.at(classIdx).c_str(), classProb);
   	putText(testImage, labels.at(classIdx), Point(20, 20), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0, 0, 255), 2, 8);
   	imshow("Image Category", testImage);
   	waitKey(0);
   	return 0;
   }
   /* 读取图像的1000个分类标记文本数据 */
   vector<String> readClasslabels() {
   	std::vector<String> classNames;
   	std::ifstream fp(labelFile);
   	if (!fp.is_open())
   	{
   		std::cerr << "File with classes labels not found: " << labelFile << std::endl;
   		exit(-1);
   	}
   	std::string name;
   	while (!fp.eof())
   	{
   		std::getline(fp, name);
   		if (name.length())
   			classNames.push_back(name.substr(name.find(' ') + 1));
   	}
   	fp.close();
   	return classNames;
   }
   
   

4. 下载模型文件
   caffemodel文件：http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel
   bvlc_googlenet.prototxt文件：https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/bvlc_googlenet.prototxt
   synset_words.txt： http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz 中的 synset_words.txt。

5. 运行程序main.cpp输出结果：
   

调用 Caffe 检测模型（MobileNetSSD_deploy.caffemodel）

重要参考链接：

• Python OpenCV 利用DNN模块进行目标检测，语义分割和风格转换
• Opencv3.3版本以上使用SSD模型实现实时摄像头对象检测

1. 使用VS2015新建 空项目。

2. 配置项目的包含目录，库目录和附加依赖项
   

3. 新建源文件 main.cpp，内容如下：

   	
   #include 
   #include 
   #include 
   
   using namespace cv;
   using namespace cv::dnn;
   using namespace std;
   
   const size_t width = 300;
   const size_t height = 300;
   const float meanVal = 127.5;
   const float scaleFactor = 0.007843f;
   const char* classNames[] = { "background",
   "aeroplane", "bicycle", "bird", "boat",
   "bottle", "bus", "car", "cat", "chair",
   "cow", "diningtable", "dog", "horse",
   "motorbike", "person", "pottedplant",
   "sheep", "sofa", "train", "tvmonitor" };
   
   String modelFile = "models/MobileNetSSD_deploy.caffemodel";
   String model_text_file = "models/MobileNetSSD_deploy.prototxt";
   
   
   int main() 
   {
   	VideoCapture capture;
   	capture.open(0);
   	namedWindow("input", CV_WINDOW_AUTOSIZE);
   	int w = capture.get(CAP_PROP_FRAME_WIDTH);
   	int h = capture.get(CAP_PROP_FRAME_HEIGHT);
   	printf("frame width : %d, frame height : %d", w, h);
   
   	// set up net
   	Net net = readNetFromCaffe(model_text_file, modelFile);
   
   	Mat frame;
   	while (capture.read(frame)) //注意：这里提供了两种模式，调用摄像头的时候把该句取消注释即可
   	while (1)
   	{
   		frame = imread("models/car.jpg");
   		imshow("input", frame);
   	//while (capture.read(frame))
   	//{
   		//预测
   		Mat inputblob = blobFromImage(frame, scaleFactor, Size(width, height), meanVal, false);
   		net.setInput(inputblob, "data");
   		Mat detection = net.forward("detection_out");
   
   		//检测
   		Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
   		float confidence_threshold = 0.3;
   		for (int i = 0; i < detectionMat.rows; i++) {
   			float confidence = detectionMat.at<float>(i, 2);
   			if (confidence > confidence_threshold) {
   				size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
   				float tl_x = detectionMat.at<float>(i, 3) * frame.cols;
   				float tl_y = detectionMat.at<float>(i, 4) * frame.rows;
   				float br_x = detectionMat.at<float>(i, 5) * frame.cols;
   				float br_y = detectionMat.at<float>(i, 6) * frame.rows;
   
   				Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
   				rectangle(frame, object_box, Scalar(0, 0, 255), 2, 8, 0);
   				putText(frame, format("%s", classNames[objIndex]), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
   			}
   		}
   		imshow("ssd-video-demo", frame);
   		char c = waitKey(5);
   		if (c == 27) 
   		{ // ESC退出
   			break;
   		}
   	}
   	capture.release();
   	waitKey(0);
   	return 0;
   }
   

4. caffemodel文件：https://drive.google.com/file/d/0B3gersZ2cHIxRm5PMWRoTkdHdHc/view?resourcekey=0-1Lpfs4EvGDeCQz12AF64hQ
   MobileNetSSD_deploy.prototxt文件：
   https://raw.githubusercontent.com/chuanqi305/MobileNet-SSD/daef68a6c2f5fbb8c88404266aa28180646d17e0/MobileNetSSD_deploy.prototxt

5. 运行main.cpp，测试结果
   

调用 Caffe 分割模型（seg.caffemodel）

重要参考链接：

• opencv dnn模块 示例(8) 语义分割 segmentation（ENet/fcn8s）

1. 使用VS2015新建 空项目。

2. 配置项目的包含目录，库目录和附加依赖项
   

3. 新建源文件 main.cpp，内容如下：

   //use opencv_dnn module for Segmentation
   
   #include 
   #include 
   
   #include 
   #include 
   #include 
   
   #include 
   
   using namespace cv;
   using namespace dnn;
   
   std::vector<std::string> classes;
   std::vector<Vec3b> colors;
   
   void showLegend();
   
   void colorizeSegmentation(const Mat &score, Mat &segm);
   
   int main(int argc, char** argv) try {
   	//	VGG - based FCN(semantical segmentation network)
   	//	ENet(lightweight semantical segmentation network)
   
   	// 根据选择的检测模型文件进行配置 
   	float confThreshold, nmsThreshold, scale;
   	cv::Scalar mean;
   	bool swapRB;
   	int inpWidth, inpHeight;
   
   	String modelPath, configPath, classesFile;
   
   	int modelType = 0;  // 0-fcn  1-enet
   
   	if (modelType == 0){
   		confThreshold = 0.5;
   		nmsThreshold = 0.4;
   		scale = 1.0;
   		mean = Scalar{ 0,0,0 };
   		swapRB = false;
   		inpWidth = 500;
   		inpHeight = 500;
   
   		modelPath = "models/fcn8s-heavy-pascal.caffemodel";
   		configPath = "models/fcn8s-heavy-pascal.prototxt";
   		classesFile = "models/object_detection_classes_pascal_voc.txt";
   	}
   	else if (modelType == 1){
   
   		confThreshold = 0.5;
   		nmsThreshold = 0.4;
   		scale = 0.00392;
   		mean = Scalar{ 0,0,0 };
   		swapRB = false;
   		inpWidth = 512;
   		inpHeight = 256;
   
   		modelPath = "models/Enet-model-best.net";
   		configPath = "";
   		classesFile = "models/enet-classes.txt";
   	}
   
   	String colorFile = "";
   
   	String framework = "";
   
   	int backendId = cv::dnn::DNN_BACKEND_OPENCV;
   	int targetId = cv::dnn::DNN_TARGET_CPU;
   
   	// Open file with classes names.
   	if (!classesFile.empty()) {
   		const std::string file = classesFile;
   		std::ifstream ifs(file.c_str());
   		if (!ifs.is_open())
   			CV_Error(Error::StsError, "File " + file + " not found");
   		std::string line;
   
   		if (modelType == 0)
   			classes.push_back("background"); //使用的是object_detection_classes，需要增加背景；   enet不需要，注释该行
   
   		while (std::getline(ifs, line)) {
   			classes.push_back(line);
   		}
   	}
   
   	if (!colorFile.empty()) {
   		const std::string file = colorFile;
   		std::ifstream ifs(file.c_str());
   		if (!ifs.is_open())
   			CV_Error(Error::StsError, "File " + file + " not found");
   		std::string line;
   		while (std::getline(ifs, line)) {
   			std::istringstream colorStr(line.c_str());
   			Vec3b color;
   			for (int i = 0; i < 3 && !colorStr.eof(); ++i)
   				colorStr >> color[i];
   			colors.push_back(color);
   		}
   	}
   
   
   	CV_Assert(!modelPath.empty());
   	//! [Read and initialize network]
   	Net net = readNet(modelPath, configPath, framework);
   	net.setPreferableBackend(backendId);
   	net.setPreferableTarget(targetId);
   	//! [Read and initialize network]
   
   	// Create a window
   	static const std::string kWinName = "Deep learning semantic segmentation in OpenCV";
   	namedWindow(kWinName, WINDOW_AUTOSIZE);
   
   	//! [Open a video file or an image file or a camera stream]
   	VideoCapture cap;
   	//cap.open("../../data/image/person.jpg");                       // pascal voc 	
   	cap.open("models/car.jpg");   // enet Cityscapes
   
   	if (!cap.isOpened()) {
   		std::cout << "VideoCapture open failed." << std::endl;
   		return 0;
   	}
   
   	//! [Open a video file or an image file or a camera stream]
   
   	// Process frames.
   	Mat frame, blob;
   	while (waitKey(1) < 0) {
   		cap >> frame;
   		if (frame.empty()) {
   			waitKey();
   			break;
   		}
   
   		//! [Create a 4D blob from a frame]
   		blobFromImage(frame, blob, scale, Size(inpWidth, inpHeight), mean, swapRB, false);
   		//! [Create a 4D blob from a frame]
   
   		//! [Set input blob]
   		net.setInput(blob);
   		//! [Set input blob]
   		//! [Make forward pass]
   		Mat score = net.forward();
   		//! [Make forward pass]
   
   		Mat segm;
   		colorizeSegmentation(score, segm);
   
   		resize(segm, segm, frame.size(), 0, 0, INTER_NEAREST);
   		addWeighted(frame, 0.5, segm, 0.5, 0.0, frame);
   
   		// Put efficiency information.
   		std::vector<double> layersTimes;
   		double freq = getTickFrequency() / 1000;
   		double t = net.getPerfProfile(layersTimes) / freq;
   		std::string label = format("Inference time: %.2f ms", t);
   		putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0));
   
   		imshow(kWinName, frame);
   
   		if (!classes.empty())
   			showLegend();
   
   	}
   	return 0;
   }
   catch (std::exception & e) {
   	std::cerr << e.what() << std::endl;
   }
   
   void colorizeSegmentation(const Mat &score, Mat &segm)
   {
   	const int rows = score.size[2];
   	const int cols = score.size[3];
   	const int chns = score.size[1];
   
   	if (colors.empty()) {
   		// Generate colors.
   		colors.push_back(Vec3b());
   		for (int i = 1; i < chns; ++i) {
   			Vec3b color;
   			for (int j = 0; j < 3; ++j)
   				color[j] = (colors[i - 1][j] + rand() % 256) / 2;
   			colors.push_back(color);
   		}
   	}
   	else if (chns != (int)colors.size()) {
   		CV_Error(Error::StsError, format("Number of output classes does not match "
   			"number of colors (%d != %zu)", chns, colors.size()));
   	}
   
   	Mat maxCl = Mat::zeros(rows, cols, CV_8UC1);
   	Mat maxVal(rows, cols, CV_32FC1, score.data);
   	for (int ch = 1; ch < chns; ch++) {
   		for (int row = 0; row < rows; row++) {
   			const float *ptrScore = score.ptr<float>(0, ch, row);
   			uint8_t *ptrMaxCl = maxCl.ptr<uint8_t>(row);
   			float *ptrMaxVal = maxVal.ptr<float>(row);
   			for (int col = 0; col < cols; col++) {
   				if (ptrScore[col] > ptrMaxVal[col]) {
   					ptrMaxVal[col] = ptrScore[col];
   					ptrMaxCl[col] = (uchar)ch;
   				}
   			}
   		}
   	}
   
   	segm.create(rows, cols, CV_8UC3);
   	for (int row = 0; row < rows; row++) {
   		const uchar *ptrMaxCl = maxCl.ptr<uchar>(row);
   		Vec3b *ptrSegm = segm.ptr<Vec3b>(row);
   		for (int col = 0; col < cols; col++) {
   			ptrSegm[col] = colors[ptrMaxCl[col]];
   		}
   	}
   }
   
   void showLegend()
   {
   	static const int kBlockHeight = 30;
   	static Mat legend;
   	if (legend.empty()) {
   		const int numClasses = (int)classes.size();
   		if ((int)colors.size() != numClasses) {
   			CV_Error(Error::StsError, format("Number of output classes does not match "
   				"number of labels (%zu != %zu)", colors.size(), classes.size()));
   		}
   		legend.create(kBlockHeight * numClasses, 200, CV_8UC3);
   		for (int i = 0; i < numClasses; i++) {
   			Mat block = legend.rowRange(i * kBlockHeight, (i + 1) * kBlockHeight);
   			block.setTo(colors[i]);
   			putText(block, classes[i], Point(0, kBlockHeight / 2), FONT_HERSHEY_SIMPLEX, 0.5, Vec3b(255, 255, 255));
   		}
   		namedWindow("Legend", WINDOW_AUTOSIZE);
   		imshow("Legend", legend);
   	}
   }
   
   
   

4. 模型文件下载地址，caffemodel和prototxt的名字必须对应：
   caffemodel：http://dl.caffe.berkeleyvision.org/fcn8s-heavy-pascal.caffemodel
   prototxt文件：https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/fcn8s-heavy-pascal.prototxt
   类别文件：https://github.com/opencv/opencv/blob/master/samples/data/dnn/object_detection_classes_pascal_voc.txt

5. 运行程序main.cpp，测试结果为
   

Opencv 系列模型地址

上述模型基本可以从下面两个链接中下载

• Caffemodel模型地址：https://github.com/opencv/opencv/blob/master/samples/dnn/models.yml
• prototxt地址：https://github.com/opencv/opencv_extra/tree/master/testdata/dnn
• 不同数据的类别名： https://github.com/opencv/opencv/tree/master/samples/data/dnn
  • action_recongnition_kinetics.txt
  • classification_classes_ILSVRC2012.txt
  • enet-classes.txt
  • object_detection_classes_coco.txt
  • object_detection_classes_pascal_voc.txt
  • object_detection_classes_yolov3.txt
• OpenCV自带dnn的Example研究（3）— object_detection
• Opencv官方YOLO教程