opencv可视化images

代码摘录 brandon 论文里面的代码

论文下载

brandon主页

#include "ImageParser.h"
#include "Visualization.h"
#include "Environment.h"
#include "Grammar.h"
#include "ModelMetadata.h"
#include "FeatureDictionary.h"
#include "ImagePyramid.h"
#include "AppearanceModel.h"
#include <opencv/highgui.h>

using namespace IP;


// ---------- ImageGrid ----------

// use default value (m_scale = 1.0f)
// width, hegit: (12, 12) - means there is 12*12 Images (Mat)
ImageGrid::ImageGrid(int width, int height, float scale) {
	m_width = width;
	m_height = height;
	
	m_scale = scale;
}

// (col, row): the index
void ImageGrid::InsertImage(cv::Mat& img, int col, int row) {
	
	cv::Mat img2 = img;
	if (img.channels() != 3) {
		img2 = cv::Mat(img.rows, img.cols, CV_8UC3);
		for (int i=0;i<img.rows;i++) {
			for (int j=0;j<img.cols;j++) {

				unsigned char val = (int)(img.at<float>(i,j)*255);
				img2.at<cv::Vec3b>(i,j) = cv::Vec3b(val,val,val);
			}
		}
	}

	if (m_scale == 1.0f) {
		m_vecImages.push_back(img2);
	} else {
		cv::Mat imgCopy;
		cv::resize(img2, imgCopy, cv::Size(Global::Round(img2.cols*m_scale), Global::Round(img2.rows*m_scale)));
		m_vecImages.push_back(imgCopy);
	}

	// 0 - numHalfOrientation or numOrientation
	// the location of the input image in the grid like (1, 0)
	m_vecLocations.push_back(cv::Point2i(col, row));

}

void ImageGrid::Clear() {
	m_vecLocations.clear();
	m_vecImages.clear();
}

cv::Mat ImageGrid::Render() {
	// get the size of the grid
	int width = m_width;
	int height = m_height;

	// to store the image size
	std::vector<int> cellWidth(width);
	std::vector<int> cellHeight(height);

	for (int i=0;i<m_vecLocations.size();i++) {
		cv::Point loc = m_vecLocations[i];

		cellWidth[loc.x] = std::max<int>(cellWidth[loc.x], m_vecImages[i].cols);
		cellHeight[loc.y] = std::max<int>(cellHeight[loc.y], m_vecImages[i].rows);
	}

	int totalHeight = 0;
	int totalWidth = 0;

	for (int i=0;i<height;i++) {
		totalHeight += cellHeight[i];
	}
	for (int i=0;i<width;i++) {
		totalWidth += cellWidth[i];
	}

	cv::Mat img(totalHeight, totalWidth, CV_8UC3);
	img.setTo(cv::Scalar(255,255,255));

	for (int i=0;i<m_vecImages.size();i++) {
		cv::Point2i loc = m_vecLocations[i];

		int offsetX = 0;
		int offsetY = 0;
		for (int j=0;j<loc.x;j++) {
			offsetX += cellWidth[j];
		}
		for (int j=0;j<loc.y;j++) {
			offsetY += cellHeight[j];
		}

		// use a bigger image to include the images in the m_vecImages
		cv::Mat& cellImg = m_vecImages[i];
		cv::Mat subImg = img(cv::Rect(offsetX, offsetY, cellImg.cols, cellImg.rows));
		cellImg.copyTo(subImg);
	}

	// cv::imshow("ImagesForImageGrid", img);
	
	return img;
}


// ---------- Visualization ----------

Visualization::Visualization(Environment* pEnvironment) : _pEnvironment(pEnvironment) {

}

cv::Mat Visualization::VisualizeParse(ParseNode* pNode, cv::Mat& frame, bool showJoints) {
	cv::Mat frame2 = frame.clone();
	VisualizeParseOnFrame(pNode, frame2, showJoints);
	return frame2;
}

void Visualization::VisualizeParseOnFrame(ParseNode* pNode, cv::Mat& frame, bool showJoints) {
	
	for (int i=0;i<pNode->childCount;i++) {
		VisualizeParseOnFrame(&pNode->children[i], frame, showJoints);
	}

	DrawParseNode(pNode, frame, showJoints);
}


//
void Visualization::DrawParseNode(ParseNode* pNode, cv::Mat& frame, bool showJoints) {

	ModelMetadata::PartImgState& imgState = _pEnvironment->GetModelMetadata()->GetPartImgState(pNode);

	cv::Point p1 = cv::Point(Global::Round(imgState.corners[0].x), Global::Round(imgState.corners[0].y));
	cv::Point p2 = cv::Point(Global::Round(imgState.corners[1].x), Global::Round(imgState.corners[1].y));
	cv::Point p3 = cv::Point(Global::Round(imgState.corners[2].x), Global::Round(imgState.corners[2].y));
	cv::Point p4 = cv::Point(Global::Round(imgState.corners[3].x), Global::Round(imgState.corners[3].y));
	
	// estimated center point ???
	cv::Point p5 = cv::Point((p1.x+p4.x)/2, (p1.y+p4.y)/2);

	cv::Scalar color(0,0,255);
	int thickness = 1;
	cv::line(frame, p1, p2, color, thickness);
	cv::line(frame, p2, p3, color, thickness);
	cv::line(frame, p3, p4, color, thickness);
	cv::line(frame, p4, p1, color, thickness);

	// draw a circle -- fullfill
	cv::circle(frame, p5, 2, color, CV_FILLED);

	if (showJoints && (pNode->childCount > 0)) {
		Production& prod = _pEnvironment->GetGrammar()->Productions(pNode->prodID);
		Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);

		for (int i=0;i<prod.relationCount;i++) {
			Relation& relation = prod.relations[i];
			ParseNode* pProxNode = relation.GetNode1(pNode);
			ParseNode* pDistNode = relation.GetNode2(pNode);

			Point2<short> jointProx = _pEnvironment->GetModelMetadata()->GetProximalJointLoc(prod.ID, i, pProxNode->prodID, pProxNode->state);
			Point2<short> jointDist = _pEnvironment->GetModelMetadata()->GetDistalJointLoc(prod.ID, i, pDistNode->prodID, pDistNode->state);

			Point2<int> centerProx(pProxNode->state.x, pProxNode->state.y);
			Point2<int> centerDist(pDistNode->state.x, pDistNode->state.y);
			Point2<int> jointProx2(jointProx.x, jointProx.y);
			Point2<int> jointDist2(jointDist.x, jointDist.y);

			cv::line(frame, cv::Point(centerProx.x, centerProx.y), cv::Point(jointProx2.x, jointProx2.y), cv::Scalar(255,0,255));
			cv::line(frame, cv::Point(centerDist.x, centerDist.y), cv::Point(jointDist2.x, jointDist2.y), cv::Scalar(0,165,255));
			cv::circle(frame, cv::Point(jointProx2.x, jointProx2.y), 3, cv::Scalar(255,0,255), 1);
			cv::circle(frame, cv::Point(jointDist2.x, jointDist2.y), 2, cv::Scalar(0,165,255), CV_FILLED);

			if (symbol.productionCount > 1) {
				std::string prodName = prod.name;
				cv::putText(frame, prodName, cv::Point(imgState.x, imgState.y), cv::FONT_HERSHEY_PLAIN, .5, cv::Scalar(255,0,0));
			}
		}

	}
}

cv::Mat Visualization::VisualizeParseFeatures(ParseNode* pNode, cv::Mat& frame, bool showNegatives) {
	cv::Mat frame2 = frame.clone();
	VisualizeParseFeaturesOnFrame(pNode, frame2, showNegatives);
	return frame2;
}

void Visualization::VisualizeParseFeaturesOnFrame(ParseNode* pNode, cv::Mat& frame, bool showNegatives) {

	DrawParseModel(pNode, frame, showNegatives);

	for (int i=0;i<pNode->childCount;i++) {
		VisualizeParseFeaturesOnFrame(&pNode->children[i], frame, showNegatives);
	}
}

void Visualization::DrawParseModel(ParseNode* pNode, cv::Mat& frame, bool showNegatives, int featureBlock) {

	Production& prod = _pEnvironment->GetGrammar()->Productions(pNode->prodID);
	Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);
	PartState& s = pNode->state;

	int levelIndex = pNode->state.scale+symbol.scaleOffset;
	float scale = 1.0f / (_pEnvironment->GetScaleFactor(levelIndex));

	AppearanceTemplate& t = _pEnvironment->GetAppearanceTemplates()->Templates(pNode->prodID);
	
	Point2<float> center(s.x,s.y);

	float maxWeight = -Global::Inf;

	for (int i=0;i<t.nodeCount;i++) {
		float nodeWeight = t.weights[i];
		
		maxWeight = std::max<float>(maxWeight, fabs(nodeWeight));
	}

	for (int i=0;i<t.nodeCount;i++) {

		float nodeWeight = t.weights[i];

		AppearanceTemplate::Element& elem = t.feat[s.orientation][i];

		Point2<int> elemLoc(elem.loc.x, elem.loc.y);

		int elemFeatureID = elem.featureID;
		int elemDeformationID = elem.deformationID;

		int elemFeatureBlock = _pEnvironment->GetFeatureDictionary()->GetFeatureInfo(elemFeatureID).blockID;
		if ((featureBlock >= 0) && (elemFeatureBlock != featureBlock)) {
			continue;
		}

		Point2<int> loc(Global::Round(center.x+elemLoc.x*scale), Global::Round(center.y+elemLoc.y*scale));

		int pixVal = (255-(int)(255*fabs(nodeWeight)/maxWeight));

		Pix24BppBGR intensity(pixVal, pixVal, pixVal);
		if (nodeWeight < 0) {
			if (!showNegatives) {
				continue;
			}
			intensity.R = 255;
		}

		_pEnvironment->GetFeatureDictionary()->DrawSymbol(frame, scale, loc, intensity, elemFeatureID, elemDeformationID);
	}
}

void Visualization::VisualizeProductions() {
	// Draw all productions in isolation with their joint locations
	std::string cacheDir = _pEnvironment->AppendPath(_pEnvironment->GetGlobalStrings().cachePath, "Visualization");
	std::string geometryDir = _pEnvironment->AppendPath(cacheDir, "Geometry");
	std::string templateDir = _pEnvironment->AppendPath(cacheDir, "Template");
	_pEnvironment->VerifyDirectoryExists(cacheDir);
	_pEnvironment->VerifyDirectoryExists(geometryDir);
	_pEnvironment->VerifyDirectoryExists(templateDir);

	int midScale = _pEnvironment->GetGlobalParameters().numScaleLevels/2;
	int numOrientations = _pEnvironment->GetGlobalParameters().numOrientations;

	// Visualize geometry
	Grammar* pGrammar = _pEnvironment->GetGrammar();
	for (int i=0;i<pGrammar->GetProductionCount();i++) {
		Production& prod = pGrammar->Productions(i);
		if (prod.childCount == 0) {
			continue;
		}

		int maxO = 0;
		int maxOWeight = -Global::Inf;
		for (int j=0;j<_pEnvironment->GetGlobalParameters().numOrientations;j++) {
			if (prod.orientationWeights[j] > maxOWeight) {
				maxOWeight = prod.orientationWeights[j];
				maxO = j;
			}
		}

		Point2<float> dim = _pEnvironment->GetModelMetadata()->GetPartDimensions(i, midScale);
		float maxDim = std::max<float>(dim.x, dim.y);
		int imgDim =Global::Round(maxDim);
		int margin = imgDim/4;
		float gridMargin = margin;
		cv::Mat frame(imgDim+2*margin, imgDim+2*margin, CV_8UC3);
		frame.setTo(cv::Scalar(255,255,255));

		ParseNode node;
		pGrammar->InitParseNode(&node, i);
		PartState s(Global::Round(gridMargin+maxDim/2), Global::Round(gridMargin+maxDim/2), maxO, midScale);
		node.state = s;
		node.prodID = i;

		for (int j=0;j<prod.childCount;j++) {
			Symbol& childSymbol = pGrammar->Symbols(prod.childSymbolIDs[j]);
			Production& childProd = pGrammar->Productions(childSymbol.productionIDs[0]);
			node.children[j].prodID = childProd.ID;
			node.children[j].state = PartState(0,0,0,midScale);
			node.children[j].childCount = 0;
		}

		for (int j=0;j<prod.relationCount;j++) {
			Relation& r = prod.relations[j];

			int maxChildO = 0;
			int maxChildOWeight = -Global::Inf;
			for (int k=0;k<numOrientations;k++) {
				if (r.orientationWeights[k] > maxChildOWeight) {
					maxChildOWeight = r.orientationWeights[k];
					maxChildO = k;
				}
			}

			ParseNode* pProxNode = r.GetNode1(&node);
			ParseNode* pDistNode = r.GetNode2(&node);
			pDistNode->state.orientation = (pProxNode->state.orientation+maxChildO) % numOrientations;

			Point2<short> jProx = _pEnvironment->GetModelMetadata()->GetProximalJointLoc(i, j, pProxNode->prodID, pProxNode->state);
			Point2<short> jDist = _pEnvironment->GetModelMetadata()->GetDistalJointLoc(i, j, pDistNode->prodID, pDistNode->state);

			pDistNode->state.x = jProx.x - jDist.x;
			pDistNode->state.y = jProx.y - jDist.y;
		}

		VisualizeParseOnFrame(&node, frame, true);

		pGrammar->ReleaseParseNode(&node);

		std::string fileName = _pEnvironment->AppendPath(geometryDir, _pEnvironment->FormatString("%s_%s.png", pGrammar->Symbols(prod.symbolID).name, prod.name));
		cv::imwrite(fileName.c_str(), frame);
	}

	// Visualize AB templates
	for (int i=0;i<pGrammar->GetProductionCount();i++) {
		Production& prod = pGrammar->Productions(i);
		Symbol& symbol = pGrammar->Symbols(prod.symbolID);

		int maxO = 0;
		float maxOWeight = -Global::Inf;
		for (int j=0;j<symbol.orientationMapLen;j++) {
			if (prod.orientationWeights[j] > maxOWeight) {
				maxOWeight = prod.orientationWeights[j];
				maxO = symbol.orientationMap[j];
			}
		}

		Point2<float> dim = _pEnvironment->GetModelMetadata()->GetPartDimensions(i, midScale);
		float maxDim = std::max<float>(dim.x, dim.y);
		int imgDim = Global::Round(maxDim);
		int margin = imgDim/4;
		float gridMargin = margin;

		ParseNode node;
		pGrammar->InitParseNode(&node, i);
		PartState s(Global::Round(gridMargin+maxDim/2), Global::Round(gridMargin+maxDim/2), maxO, midScale);
		node.state = s;
		node.prodID = i;

		int featureBlocks = _pEnvironment->GetFeatureDictionary()->GetFeatureBlockCount();
		ImageGrid grid(featureBlocks, 1);
		for (int j=0;j<featureBlocks;j++) {
			cv::Mat frame(imgDim+2*margin, imgDim+2*margin, CV_8UC3);
			frame.setTo(cv::Scalar(255,255,255));
			DrawParseModel(&node, frame, true, j);
			//DrawParseFeatureEdges(&node, frame);
			grid.InsertImage(frame,j,0);
		}

		pGrammar->ReleaseParseNode(&node);

		std::string fileName = _pEnvironment->AppendPath(templateDir, _pEnvironment->FormatString("%s_%s.png", pGrammar->Symbols(prod.symbolID).name, prod.name));
		cv::imwrite(fileName.c_str(), grid.Render());
	}
}

cv::Mat Visualization::VisualizeMatrix2D(Matrix2D<float>& mat) {
	cv::Mat vis(mat.Dim2(), mat.Dim1(), CV_8UC3);
	float maxScore = mat(mat.GetMaxLoc());
	float minScore = mat(mat.GetMinLoc());
	if (minScore == maxScore) {
		if (minScore > 0) {
			minScore = 0;
		} else if (maxScore < 0) {
			maxScore = 0;
		} else {
			vis.setTo(cv::Scalar(0,0,0));
			return vis;
		}
	}
	for (int x=0;x<mat.Dim1();x++) {
		for (int y=0;y<mat.Dim2();y++) {
			char score = (char)(255*(mat(x,y)-minScore)/(maxScore-minScore));
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(score, score, score);
		}
	}
	return vis;
}

cv::Mat Visualization::VisualizeMatrix2D(Matrix2D<double>& mat) {
	cv::Mat vis(mat.Dim2(), mat.Dim1(), CV_8UC3);
	double maxScore = mat(mat.GetMaxLoc());
	double minScore = mat(mat.GetMinLoc());
	if (minScore == maxScore) {
		if (minScore > 0) {
			minScore = 0;
		} else if (maxScore < 0) {
			maxScore = 0;
		} else {
			vis.setTo(cv::Scalar(0,0,0));
			return vis;
		}
	}
	for (int x=0;x<mat.Dim1();x++) {
		for (int y=0;y<mat.Dim2();y++) {
			char score = (char)(255*(mat(x,y)-minScore)/(maxScore-minScore));
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(score, score, score);
		}
	}
	return vis;
}

cv::Mat Visualization::VisualizeMatrix2D(Matrix2D<unsigned char>& mat) {
	cv::Mat vis(mat.Dim2(), mat.Dim1(), CV_8UC3);
	unsigned char maxScore = mat(mat.GetMaxLoc());
	unsigned char minScore = mat(mat.GetMinLoc());
	if (minScore == maxScore) {
		if (minScore > 0) {
			minScore = 0;
		} else if (maxScore < 0) {
			maxScore = 0;
		} else {
			vis.setTo(cv::Scalar(0,0,0));
			return vis;
		}
	}
	for (int x=0;x<mat.Dim1();x++) {
		for (int y=0;y<mat.Dim2();y++) {
			unsigned char score = (unsigned char)(255*(mat(x,y)-minScore)/(maxScore-minScore));
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(score, score, score);
		}
	}
	return vis;
}

cv::Mat Visualization::VisualizeMatrix2D(Matrix2D<Pix24BppBGR>& mat) {
	cv::Mat vis(mat.Dim2(), mat.Dim1(), CV_8UC3);
	for (int x=0;x<mat.Dim1();x++) {
		for (int y=0;y<mat.Dim2();y++) {
			Pix24BppBGR& pix = mat(x,y);
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(pix.B, pix.G, pix.R);
		}
	}
	return vis;
}

cv::Mat Visualization::VisualizeStateMap(StateMap<float>& map) {
	ImageGrid grid(map.NumOrientations(), map.NumScales());
	for (int o=0;o<map.NumOrientations();o++) {
		for (int s=0;s<map.NumScales();s++) {
			cv::Mat vis = VisualizeStateMap(map, o, s, -Global::Inf, Global::Inf);
			grid.InsertImage(vis, o, s);
		}
	}
	return grid.Render();
}

cv::Mat Visualization::VisualizeStateMap(StateMap<float>& map, int o, int s) {
	cv::Mat vis(map.Height(), map.Width(), CV_8UC3);
	float maxScore = map(map.GetMaxLoc());
	float minScore = map(map.GetMinLoc());
	if (minScore == maxScore) {
		if (minScore > 0) {
			minScore = 0;
		} else if (maxScore < 0) {
			maxScore = 0;
		} else {
			vis.setTo(cv::Scalar(0,0,0));
			return vis;
		}
	}
	for (int x=0;x<map.Width();x++) {
		for (int y=0;y<map.Height();y++) {
			char score = (char)(255*(map(x,y,o,s)-minScore)/(maxScore-minScore));
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(score, score, score);
		}
	}
	return vis;
}

cv::Mat Visualization::VisualizeStateMap(StateMap<float>& map, int o, int s, float minBounds, float maxBounds) {
	cv::Mat vis(map.Height(), map.Width(), CV_8UC3);
	float minScore = maxBounds;
	float maxScore = minBounds;
	for (int x=0;x<map.Width();x++) {
		for (int y=0;y<map.Height();y++) {
			float score = map(x,y,o,s);
			if (Global::CheckInclusiveBounds<float>(score, minBounds, maxBounds)) {
				minScore = std::min<float>(minScore, score);
				maxScore = std::max<float>(maxScore, score);
			}
		}
	}
	if (minScore == maxScore) {
		if (minScore > 0) {
			minScore = 0;
		} else if (maxScore < 0) {
			maxScore = 0;
		} else {
			vis.setTo(cv::Scalar(0,0,0));
			return vis;
		}
	}
	for (int x=0;x<map.Width();x++) {
		for (int y=0;y<map.Height();y++) {
			int score = (int)Global::Box<float>((255*(map(x,y,o,s)-minScore)/(maxScore-minScore)), 0, 255);
			vis.at<cv::Vec3b>(y,x) = cv::Vec3b(score, score, score);
		}
	}
	return vis;
}

//
void Visualization::LightenImage(cv::Mat& frame) {
	frame = ((255*.6) + frame * 0.4);
}

void Visualization::FlipImageH(cv::Mat& frame) {
	// Hard coded for BGR24bpp images
	cv::Vec3b temp;
	for (int y=0;y<frame.rows;y++) {
		for (int x=0;x<frame.cols/2;x++) {
			temp = frame.at<cv::Vec3b>(y,x);
			frame.at<cv::Vec3b>(y,x) = frame.at<cv::Vec3b>(y,frame.cols-x-1);
			frame.at<cv::Vec3b>(y,frame.cols-x-1) = temp;
		}
	}
}

TreeRasterNode* Visualization::BuildRasterTree(ParseNode* pParse, float spacingX, float spacingY) {

	TreeRasterNode* pRasterNode = new TreeRasterNode(pParse, pParse->childCount);
	pRasterNode->offset = Point2<float>(0,0);

	int childrenWidth = 0;
	for (int i=0;i<pParse->childCount;i++) {
		TreeRasterNode* pChildRasterNode = BuildRasterTree(&pParse->children[i], spacingX, spacingY);
		pRasterNode->children[i] = pChildRasterNode;
		childrenWidth += pChildRasterNode->width;
	}

	pRasterNode->width = std::max<int>(0, pRasterNode->childCount-1) + childrenWidth;
	float totalWidth = pRasterNode->width*spacingX;
	float startX = -1.0f * (totalWidth/2);
	for (int i=0;i<pRasterNode->childCount;i++) {
		float childWidth = (pRasterNode->children[i]->width*spacingX);
		pRasterNode->children[i]->offset.x = startX+childWidth/2;
		pRasterNode->children[i]->offset.y = spacingY;
		startX += (pRasterNode->children[i]->width+1)*spacingX;
	}
	pRasterNode->rasterWidth = totalWidth;
	return pRasterNode;
}

void Visualization::DrawParseTree(TreeRasterNode* pRasterNode, cv::Mat& frame, Point2<int> location) {
	// NOTE: drawing code here is just for test

	for (int i=0;i<pRasterNode->childCount;i++) {
		Point2<int> childLocation = location;

		childLocation.x += Global::Round(pRasterNode->children[i]->offset.x);
		childLocation.y += Global::Round(pRasterNode->children[i]->offset.y);

		cv::line(frame, cv::Point(location.x, location.y), cv::Point(childLocation.x, childLocation.y), cv::Scalar(0,255,0));
		DrawParseTree(pRasterNode->children[i], frame, childLocation);
	}

	cv::circle(frame, cv::Point(location.x, location.y), 5, cv::Scalar(0,255,0));
}

cv::Mat Visualization::VisualizeHistogram(Histogram& h) {
	int width = 500;
	int height = 100;
	cv::Mat frame(height, width, CV_8UC3);
	frame.setTo(cv::Scalar(255,255,255));
	for (int i=0;i<h.GetBinCount()-1;i++) {
		int x1 = (width*i)/h.GetBinCount();
		int x2 = (width*(i+1))/h.GetBinCount();

		int y1 = height-1-Global::Round(height*h(i));
		int y2 = height-1-Global::Round(height*h(i+1));
		cv::line(frame, cv::Point(x1,y1), cv::Point(x2,y2), cv::Scalar(0,255,0));
	}
	return frame;
}

cv::Mat Visualization::VisualizeHistograms(Histogram& h1, Histogram& h2) {
	int width = 500;
	int height = 100;
	cv::Mat frame(height, width, CV_8UC3);
	frame.setTo(cv::Scalar(255,255,255));

	for (int i=0;i<h1.GetBinCount()-1;i++) {
		int x1 = (width*i)/h1.GetBinCount();
		int x2 = (width*(i+1))/h1.GetBinCount();
		int y1 = height-1-Global::Round(height*h1(i));
		int y2 = height-1-Global::Round(height*h1(i+1));
		cv::line(frame, cv::Point(x1,y1), cv::Point(x2,y2), cv::Scalar(0,255,0));
	}

	for (int i=0;i<h2.GetBinCount()-1;i++) {
		int x1 = (width*i)/h2.GetBinCount();
		int x2 = (width*(i+1))/h2.GetBinCount();
		int y1 = height-1-Global::Round(height*h2(i));
		int y2 = height-1-Global::Round(height*h2(i+1));
		cv::line(frame, cv::Point(x1,y1), cv::Point(x2,y2), cv::Scalar(0,0,255));
	}
	return frame;
}


Pix24BppBGR Visualization::JetColormap(float val) {
	// 'jet' colormap
	float r = 1.8-5.5*fabs(val-.79);
	float g = 1.8-5.5*fabs(val-.5);
	float b = 1.8-5.5*fabs(val-.21);

	Pix24BppBGR pix;
	pix.R = Global::Box<float>(255*r, 0, 255);
	pix.G = Global::Box<float>(255*g, 0, 255);
	pix.B = Global::Box<float>(255*b, 0, 255);
	return pix;
}


//use dfs to get the parse node in parse tree
void Visualization::FlattenParse(ParseNode* pNode, std::vector<ParseNode*>& nodes) {
	nodes.push_back(pNode);
	for (int i=0;i<pNode->childCount;i++) {
		FlattenParse(&pNode->children[i], nodes);
	}
}

void Visualization::CropPart(ParseNode* pNode, cv::Mat& frame, Size2<int> templateDim, Matrix2D<Pix24BppBGR>& output) {
	output.Init(templateDim);
	Point2<int> dimH((templateDim.width-1)/2, (templateDim.height-1)/2);

	PartState& s = pNode->state;
	Production& prod = _pEnvironment->GetGrammar()->Productions(pNode->prodID);
	Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);
	float levelScale = _pEnvironment->GetScaleFactor(s.scale+symbol.scaleOffset);

	cv::Mat frame2;
	int levelWidth = Global::Round(frame.cols*levelScale);
	int levelHeight = Global::Round(frame.rows*levelScale);
	cv::resize(frame, frame2, cv::Size(levelWidth, levelHeight));

	for (int dx=-dimH.x;dx<=dimH.x;dx++) {
		for (int dy=-dimH.y;dy<=dimH.y;dy++) {
			float xif = levelScale*s.x+_pEnvironment->GetDX(s.orientation, dx, dy);
			float yif = levelScale*s.y+_pEnvironment->GetDY(s.orientation, dx, dy);

			int xi = (int)floor(xif);
			int yi = (int)floor(yif);

			if (!Global::CheckBounds<int>(xi, 0, levelWidth-2) || !Global::CheckBounds<int>(yi, 0, levelHeight-2)) {
				continue;
			}

			float deltaX = xif - xi;
			float deltaY = yif - yi;
			float w1 = (1-deltaX)*(1-deltaY);
			float w2 = (deltaX)*(1-deltaY);
			float w3 = (1-deltaX)*(deltaY);
			float w4 = (deltaX)*(deltaY);

			cv::Vec3b p1 = frame2.at<cv::Vec3b>(yi,xi);
			cv::Vec3b p2 = frame2.at<cv::Vec3b>(yi,xi+1);
			cv::Vec3b p3 = frame2.at<cv::Vec3b>(yi+1,xi);
			cv::Vec3b p4 = frame2.at<cv::Vec3b>(yi+1,xi+1);

			Pix24BppBGR pix;
			pix.B = (unsigned char)(w1*p1[0]+w2*p2[0]+w3*p3[0]+w4*p4[0]);
			pix.G = (unsigned char)(w1*p1[1]+w2*p2[1]+w3*p3[1]+w4*p4[1]);
			pix.R = (unsigned char)(w1*p1[2]+w2*p2[2]+w3*p3[2]+w4*p4[2]);

			output(dimH.x+dx,dimH.y+dy) = pix;
		}
	}

}


void Visualization::VisualizeMeanImages(DataSet& dataSet, bool saveCrops) {
	std::string visOutputDir = _pEnvironment->AppendPath(_pEnvironment->GetGlobalStrings().cachePath, "Visualization");
	std::string visPooledDir = _pEnvironment->AppendPath(visOutputDir, "MeanImages");
	_pEnvironment->VerifyDirectoryExists(visOutputDir);
	_pEnvironment->VerifyDirectoryExists(visPooledDir);

	int productionCount = _pEnvironment->GetGrammar()->GetProductionCount();
 	std::vector<std::string> cropDirs(productionCount);

	if (saveCrops) {
		std::string cropsDir = _pEnvironment->AppendPath(visPooledDir, "Crops");
		_pEnvironment->VerifyDirectoryExists(cropsDir);
		for (int i=0;i<productionCount;i++) {
			Production& prod = _pEnvironment->GetGrammar()->Productions(i);
			Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);

			std::string prodName = _pEnvironment->FormatString("%s_%s", symbol.name, prod.name);
			std::string prodCropDir = _pEnvironment->AppendPath(cropsDir, prodName);
			cropDirs[i] = prodCropDir;
			_pEnvironment->VerifyDirectoryExists(prodCropDir);
		}
	}


	Point2<int>* dims = new Point2<int>[productionCount];
	Point2<int>* dimsH = new Point2<int>[productionCount];
	Matrix2D<int>* meanR = new Matrix2D<int>[productionCount];
	Matrix2D<int>* meanG = new Matrix2D<int>[productionCount];
	Matrix2D<int>* meanB = new Matrix2D<int>[productionCount];

	int* counts = new int[productionCount];
	int* cropCounts = new int[productionCount];
	memset(counts, 0x00, sizeof(int)*productionCount);
	memset(cropCounts, 0x00, sizeof(int)*productionCount);

	for (int i=0;i<productionCount;i++) {

		Production& prod = _pEnvironment->GetGrammar()->Productions(i);
		Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);

		ModelMetadata::SymbolStats& stats = _pEnvironment->GetModelMetadata()->GetStats(prod.symbolID);

		float templateScaleFactor = _pEnvironment->GetScaleFactor(symbol.scaleOffset);
		float templateLenF = templateScaleFactor * stats.meanLength * _pEnvironment->GetModelMetadata()->GetGlobals().referenceLength;
		float templateWidthF = templateLenF / stats.meanAspect;

		// Scale the dimensions to add a small margin
		int margin = 10;
		int length = Global::Round(templateLenF+margin);
		int width = Global::Round(templateWidthF+margin);
		if ((length % 2) == 0) { length++; }
		if ((width % 2) == 0) { width++; }

		// mean - image patch for the part
		dims[i] = Point2<int>(length, width);
		dimsH[i] = Point2<int>(length/2, width/2);

		meanR[i].Init(dims[i].x, dims[i].y);
		meanG[i].Init(dims[i].x, dims[i].y);
		meanB[i].Init(dims[i].x, dims[i].y);
	}

	for (int iSample=0;iSample<dataSet.count;iSample++) {

		std::vector<ParseNode*> nodes;
		FlattenParse(&dataSet.parses[iSample], nodes);

		cv::Mat frame = cv::imread(dataSet.paths[iSample]);

		for (int i=0;i<nodes.size();i++) {

			ParseNode* pNode = nodes[i];
			PartState& s = pNode->state;

			//if (pNode->metadata.annoOcclusionState != PartMetadata::NoOcclusion) {
			//	continue;
			//}

			Production& prod = _pEnvironment->GetGrammar()->Productions(pNode->prodID);
			Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);

			Point2<int> dim = dims[pNode->prodID];
			Point2<int> dimH = dimsH[pNode->prodID];

			float levelScale = _pEnvironment->GetScaleFactor(s.scale+symbol.scaleOffset);

			cv::Mat frame2;
			int levelWidth = Global::Round(frame.cols*levelScale);
			int levelHeight = Global::Round(frame.rows*levelScale);
			cv::resize(frame, frame2, cv::Size(levelWidth, levelHeight));

			cv::Mat crop(dim.y, dim.x, CV_8UC3);
			crop.setTo(cv::Vec3b(255,255,255));

			for (int dx=-dimH.x;dx<=dimH.x;dx++) {
				for (int dy=-dimH.y;dy<=dimH.y;dy++) {

					// ??????????
					float xif = levelScale*s.x+_pEnvironment->GetDX(s.orientation, dx, dy);
					float yif = levelScale*s.y+_pEnvironment->GetDY(s.orientation, dx, dy);

					int xi = (int)floor(xif);
					int yi = (int)floor(yif);

					if (!Global::CheckBounds<int>(xi, 0, levelWidth-2) || !Global::CheckBounds<int>(yi, 0, levelHeight-2)) {
						continue;
					}

					float deltaX = xif - xi;
					float deltaY = yif - yi;
					float w1 = (1-deltaX)*(1-deltaY);
					float w2 = (deltaX)*(1-deltaY);
					float w3 = (1-deltaX)*(deltaY);
					float w4 = (deltaX)*(deltaY);

					cv::Vec3b p1 = frame2.at<cv::Vec3b>(yi,xi);
					cv::Vec3b p2 = frame2.at<cv::Vec3b>(yi,xi+1);
					cv::Vec3b p3 = frame2.at<cv::Vec3b>(yi+1,xi);
					cv::Vec3b p4 = frame2.at<cv::Vec3b>(yi+1,xi+1);
					int pix1 = (int)(w1*p1[0]+w2*p2[0]+w3*p3[0]+w4*p4[0]);
					int pix2 = (int)(w1*p1[1]+w2*p2[1]+w3*p3[1]+w4*p4[1]);
					int pix3 = (int)(w1*p1[2]+w2*p2[2]+w3*p3[2]+w4*p4[2]);
					cv::Vec3b p0(pix1,pix2,pix3);

					crop.at<cv::Vec3b>(dimH.y+dy, dimH.x+dx) = p0;

					meanB[pNode->prodID](dimH.x+dx,dimH.y+dy) += pix1;
					meanG[pNode->prodID](dimH.x+dx,dimH.y+dy) += pix2;
					meanR[pNode->prodID](dimH.x+dx,dimH.y+dy) += pix3;
				}
			}

			counts[pNode->prodID]++;

			if (saveCrops) {
				std::string cropFileName = _pEnvironment->FormatString("%i.png", cropCounts[pNode->prodID]);
				cv::imwrite(_pEnvironment->AppendPath(cropDirs[pNode->prodID], cropFileName), crop);
				cropCounts[pNode->prodID]++;
			}
		}

	}

	for (int i=0;i<productionCount;i++) {
		Production& prod = _pEnvironment->GetGrammar()->Productions(i);
		Symbol& symbol = _pEnvironment->GetGrammar()->Symbols(prod.symbolID);
		int count = counts[i];
		if (count == 0) {
			continue;
		}
		cv::Mat meanImg(dims[i].y,dims[i].x,CV_8UC3);
		for (int x=0;x<dims[i].x;x++) {
			for (int y=0;y<dims[i].y;y++) {
				meanImg.at<cv::Vec3b>(y,x) = cv::Vec3b(meanB[i](x,y)/count, meanG[i](x,y)/count, meanR[i](x,y)/count);
			}
		}
		std::string fileName = _pEnvironment->AppendPath(visPooledDir, _pEnvironment->FormatString("%s_%s_mean.png", symbol.name, prod.name));
		cv::imwrite(fileName, meanImg);
	}


	delete[] dims;
	delete[] dimsH;
	delete[] meanR;
	delete[] meanG;
	delete[] meanB;
	delete[] counts;
	delete[] cropCounts;
}

摘录brandon的代码