opencv dnn模块 示例(8) 语义分割 segmentation（ENet/fcn8s）

一、opencv的示例模型文件

opencv的dnn模块读取models.yml文件中包含的目标检测模型有2种，
ENet road scene segmentation network from https://github.com/e-lab/ENet-training
Works fine for different input sizes.

• enet:
  model: “Enet-model-best.net”
  mean: [0, 0, 0]
  scale: 0.00392
  width: 512
  height: 256
  rgb: true
  classes: “enet-classes.txt”
  sample: “segmentation”
• fcn8s:
  model: “fcn8s-heavy-pascal.caffemodel”
  config: “fcn8s-heavy-pascal.prototxt”
  mean: [0, 0, 0]
  scale: 1.0
  width: 500
  height: 500
  rgb: false
  sample: “segmentation”

二、示例代码

#include 
#include 

#include 
#include 
#include 

#include 

using namespace cv;
using namespace dnn;

std::vector<std::string> classes;
std::vector<Vec3b> colors;

void showLegend();

void colorizeSegmentation(const Mat &score, Mat &segm);

int main(int argc, char** argv) try {
	//	VGG - based FCN(semantical segmentation network)
	//	ENet(lightweight semantical segmentation network)

	// 根据选择的检测模型文件进行配置 
	float confThreshold, nmsThreshold, scale;
	cv::Scalar mean;
	bool swapRB;
	int inpWidth, inpHeight;

	String modelPath, configPath, classesFile;

	int modelType = 1;  // 0-fcn  1-enet

	if (modelType == 0){
		confThreshold = 0.5;
		nmsThreshold = 0.4;
		scale = 1.0;
		mean = Scalar{ 0,0,0 };
		swapRB = false;
		inpWidth = 500;
		inpHeight = 500;

		modelPath = "../../data/testdata/dnn/fcn8s-heavy-pascal.caffemodel";
		configPath = "../../data/testdata/dnn/fcn8s-heavy-pascal.prototxt";
		classesFile = "../../data/dnn/object_detection_classes_pascal_voc.txt";
	}
	else if (modelType == 1){

		confThreshold = 0.5;
		nmsThreshold = 0.4;
		scale = 0.00392;
		mean = Scalar{ 0,0,0 };
		swapRB = false;
		inpWidth = 512;
		inpHeight = 256;

		modelPath = "../../data/testdata/dnn/Enet-model-best.net";
		configPath = "";
		classesFile = "../../data/dnn/enet-classes.txt";
	}

	String colorFile = "";

	String framework = "";

	int backendId = cv::dnn::DNN_BACKEND_OPENCV;
	int targetId = cv::dnn::DNN_TARGET_CPU;

	// Open file with classes names.
	if (!classesFile.empty()) {
		const std::string file = classesFile;
		std::ifstream ifs(file.c_str());
		if (!ifs.is_open())
			CV_Error(Error::StsError, "File " + file + " not found");
		std::string line;

		if (modelType == 0)
			classes.push_back("background"); //使用的是object_detection_classes，需要增加背景；   enet不需要，注释该行

		while (std::getline(ifs, line)) {
			classes.push_back(line);
		}
	}

	if (!colorFile.empty()) {
		const std::string file = colorFile;
		std::ifstream ifs(file.c_str());
		if (!ifs.is_open())
			CV_Error(Error::StsError, "File " + file + " not found");
		std::string line;
		while (std::getline(ifs, line)) {
			std::istringstream colorStr(line.c_str());
			Vec3b color;
			for (int i = 0; i < 3 && !colorStr.eof(); ++i)
				colorStr >> color[i];
			colors.push_back(color);
		}
	}


	CV_Assert(!modelPath.empty());
	//! [Read and initialize network]
	Net net = readNet(modelPath, configPath, framework);
	net.setPreferableBackend(backendId);
	net.setPreferableTarget(targetId);
	//! [Read and initialize network]

	// Create a window
	static const std::string kWinName = "Deep learning semantic segmentation in OpenCV";
	namedWindow(kWinName, WINDOW_AUTOSIZE);

	//! [Open a video file or an image file or a camera stream]
	VideoCapture cap;
	//cap.open("../../data/image/person.jpg");                       // pascal voc 	
	cap.open("G:/Datasets/Cityscapes/aachen_%06d_000019.jpg");   // enet Cityscapes

	if (!cap.isOpened()) {
		std::cout << "VideoCapture open failed." << std::endl;
		return 0;
	}

	//! [Open a video file or an image file or a camera stream]

	// Process frames.
	Mat frame, blob;
	while (waitKey(1) < 0) {
		cap >> frame;
		if (frame.empty()) {
			waitKey();
			break;
		}

		//! [Create a 4D blob from a frame]
		blobFromImage(frame, blob, scale, Size(inpWidth, inpHeight), mean, swapRB, false);
		//! [Create a 4D blob from a frame]

		//! [Set input blob]
		net.setInput(blob);
		//! [Set input blob]
		//! [Make forward pass]
		Mat score = net.forward();
		//! [Make forward pass]

		Mat segm;
		colorizeSegmentation(score, segm);

		resize(segm, segm, frame.size(), 0, 0, INTER_NEAREST);
		addWeighted(frame, 0.1, segm, 0.9, 0.0, frame);

		// Put efficiency information.
		std::vector<double> layersTimes;
		double freq = getTickFrequency() / 1000;
		double t = net.getPerfProfile(layersTimes) / freq;
		std::string label = format("Inference time: %.2f ms", t);
		putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0));

		imshow(kWinName, frame);

		if (!classes.empty())
			showLegend();

	}
	return 0;
}
catch (std::exception & e) {
	std::cerr << e.what() << std::endl;
}

void colorizeSegmentation(const Mat &score, Mat &segm)
{
	const int rows = score.size[2];
	const int cols = score.size[3];
	const int chns = score.size[1];

	if (colors.empty()) {
		// Generate colors.
		colors.push_back(Vec3b());
		for (int i = 1; i < chns; ++i) {
			Vec3b color;
			for (int j = 0; j < 3; ++j)
				color[j] = (colors[i - 1][j] + rand() % 256) / 2;
			colors.push_back(color);
		}
	}
	else if (chns != (int)colors.size()) {
		CV_Error(Error::StsError, format("Number of output classes does not match "
			"number of colors (%d != %zu)", chns, colors.size()));
	}

	Mat maxCl = Mat::zeros(rows, cols, CV_8UC1);
	Mat maxVal(rows, cols, CV_32FC1, score.data);
	for (int ch = 1; ch < chns; ch++) {
		for (int row = 0; row < rows; row++) {
			const float *ptrScore = score.ptr<float>(0, ch, row);
			uint8_t *ptrMaxCl = maxCl.ptr<uint8_t>(row);
			float *ptrMaxVal = maxVal.ptr<float>(row);
			for (int col = 0; col < cols; col++) {
				if (ptrScore[col] > ptrMaxVal[col]) {
					ptrMaxVal[col] = ptrScore[col];
					ptrMaxCl[col] = (uchar)ch;
				}
			}
		}
	}

	segm.create(rows, cols, CV_8UC3);
	for (int row = 0; row < rows; row++) {
		const uchar *ptrMaxCl = maxCl.ptr<uchar>(row);
		Vec3b *ptrSegm = segm.ptr<Vec3b>(row);
		for (int col = 0; col < cols; col++) {
			ptrSegm[col] = colors[ptrMaxCl[col]];
		}
	}
}

void showLegend()
{
	static const int kBlockHeight = 30;
	static Mat legend;
	if (legend.empty()) {
		const int numClasses = (int)classes.size();
		if ((int)colors.size() != numClasses) {
			CV_Error(Error::StsError, format("Number of output classes does not match "
				"number of labels (%zu != %zu)", colors.size(), classes.size()));
		}
		legend.create(kBlockHeight * numClasses, 200, CV_8UC3);
		for (int i = 0; i < numClasses; i++) {
			Mat block = legend.rowRange(i * kBlockHeight, (i + 1) * kBlockHeight);
			block.setTo(colors[i]);
			putText(block, classes[i], Point(0, kBlockHeight / 2), FONT_HERSHEY_SIMPLEX, 0.5, Vec3b(255, 255, 255));
		}
		namedWindow("Legend", WINDOW_AUTOSIZE);
		imshow("Legend", legend);
	}
}

3、示例

(1) fcn8s-heavy-pascal 测试结果

person.jpg 原图，图例，结果图如下 （opencl 比 cpu慢2倍…）

另外两张图结果

（2）Enet-model 测试结果

（opencl 比 cpu快3倍… 无语…）