图像融合之多波段融合（Multiband Blending）/拉普拉斯金字塔融合（Laplacian Pyramid Blending）

多波段融合又叫拉普拉斯金字塔融合。

多波段融合的思想是对待融合的图像分别构建拉普拉斯金字塔，（拉普拉斯算子可以提取出图像的高频信息，在拉普拉斯金字塔中，越往上层的图像越高频）然后对同一层图像按照某种规则融合，一般是Alpha blending/Feathering；对于不同层图像（不同频率段的图像）进行不同规则的融合，高频部分blend slowly，低频部分blend quickly；对融合后的图像金字塔重建得到最终结果图[1][2]。

 

Alpha Blending/Feathering:

如何blend：1. 直接对overlap区域average；2.找到center seam然后blur seam

这里最关键点是如何设置window size，window size的设置会对融合效果影响很大：

最佳window应该使融合后的图像smooth but not ghosted。在上面就是第三副图。

从图像结构特征上考虑，最佳window的约束是这样的：

1.避免产生缝需要：window = size of largest prominent feature

2.避免产生鬼影需要：window <= 2*size of smallest prominent feature

 

在频域上上述约束就很好考虑了，融合窗的最大频率要<= 2*size of smallest frequency。

如果信号的带宽很小，频率都集中在一小块，很好做决定，但是图像的频率带宽很大，分布范围很广，这时候就不好确定窗的大小了。这个问题的解决方法就是分频带（band）进行Feather。这种方法在空间域就能方便的实现，就是使用带通滤波器（或者等效带通滤波器的方法）作用于图像得到Bandpass Images。

Pyramid Blending就是其中一种优秀的实现方法。

 

The Laplacian Pyramid：

LOG的实现依然是用DOG去近似。

 

拉普拉斯金字塔进行blending的步骤如下：

1. Build Laplacian pyramids LA and LB from images A and B ；

2. Build a Gaussian pyramid GM from selection mask M；

3. Form a combined pyramid LS from LA and LB using nodes of GM as weights:

        LS = GM * LA + (1-GM) * LB

4. Collapse the LS pyramid to get the final blended image

 

一种Matlab实现[3]，非常棒，条理清晰：

测试图片在此：

链接：https://pan.baidu.com/s/1MNp1zjj5i7Z3Okw6btkhAw 
提取码：6oe0 

注意这里处理的是灰度图片。

clear all
close all
clc
%% Read the images, assuming images to be of same size
A=imread('images2\ghost rider.jpg');
B=imread('images2\nicholas cage.jpg');
height=size(A,1);
width=size(A,2);

%% Convert images to grayscale
imgA=rgb2gray(A);
imgB=rgb2gray(B);

%% Create mask
partition=floor(width/2);
mask=ones(height,width);
mask(:,1:partition)=0*mask(:,1:partition);
% figure, imshow(uint8(mask*255));

%% Create gaussian pyramids for both the images and the mask
levels=floor(log2(min([width, height])));
sigma=2;
hsize=3*sigma+1;
sigmaMask=10;
hsizeMask=3*sigmaMask+1;
gaussPyrA=gaussianPyramid(imgA,levels,sigma,hsize)
gaussPyrB=gaussianPyramid(imgB,levels,sigma,hsize)
gaussPyrMask=gaussianPyramid(mask,levels,sigmaMask,hsizeMask)
% for i=1:levels
%     figure,imshow(uint8(gaussPyrMask{i,1}*255));
% end

%% Create Laplacian pyramids from both the images 
laplacePyrA=laplacianPyramid(gaussPyrA);
laplacePyrB=laplacianPyramid(gaussPyrB);

%% Blend laplacian pyramid
blendedPyr=blendPyramid(laplacePyrA,laplacePyrB,gaussPyrMask);

%% Reconstruct image from blended pyramid
blendedImg=collapsePyramid(blendedPyr);
imshow(blendedImg);

对应各步骤的函数：

function pyr=gaussianPyramid(A,levels,sigma,hsize)
pyr=cell(levels,1);
h=fspecial('gaussian',hsize,sigma);
pyr{1,1}=A;
for i=2:levels
    temp=imfilter(pyr{i-1,1},h,'symmetric','corr');
    pyr{i,1}=temp(1:2:end,1:2:end);
end

function pyr=laplacianPyramid(A)
levels=size(A,1)
pyr=cell(levels,1);
pyr{levels,1}=double(A{levels,1});
for i=levels-1:-1:1
    temp=imresize(A{i+1,1},size(A{i,1}));
    pyr{i,1}=double(A{i,1})-double(temp);
end

function pyr=blendPyramid(pyrA,pyrB,pyrMask)
levels=size(pyrMask,1);
pyr=cell(levels,1);
for l=1:levels
    pyr{l,1}=(1-pyrMask{l,1}).*pyrA{l,1}+(pyrMask{l,1}).*pyrB{l,1};
end

function img=collapsePyramid(A)
levels=size(A,1)
for i=levels-1:-1:1
    temp=imresize(uint8(A{i+1,1}),size(A{i,1}));
    A{i,1}=A{i,1}+double(temp);
end
img=uint8(A{1,1});

下面是我仿照上面的Matlab 写的基于OpenCV的C++实现：

测试图也是上面的链接，同样也是处理灰度图。

#include 
#include 
#include 
#include 
#include 

using namespace std;
using namespace cv;

// show image
void showImage(String windowName, Mat img, int flag = 0)
{
	namedWindow(windowName, CV_WINDOW_NORMAL);//0: CV_WINDOW_NORMAL 1: WINDOW_AUTOSIZE
	imshow(windowName, img);
	waitKey(0);
	//destroyWindow(windowName);
}

vector> gaussianPyramid(Mat img, int levels)
{
	vector> _gaussianPyramid;
	_gaussianPyramid.push_back(img);
	Mat currentImage = img;
	//cout << currentImage.size() << endl;
	for (int i = 1; i < levels; i++)
	{
		Mat downsampleImage;
		pyrDown(currentImage, downsampleImage); // Blurs an image and downsamples it.
		_gaussianPyramid.push_back(downsampleImage);
		//showImage("currentImage", currentImage);
		//cout << downsampleImage.size() << endl;
		currentImage = downsampleImage;
	}
	return _gaussianPyramid;
}


vector> laplacianPyramid(vector> gaussPyrImg)
{
	int levels = gaussPyrImg.size();
	vector> _laplacianPyramid;
	_laplacianPyramid.push_back(gaussPyrImg[levels - 1]);// order reverse !!
	//cout << gaussPyrImg[levels - 1].size() << endl;
	for (int i = levels - 2 ; i >= 0; i--)
	{
		Mat upsampleImage;
		pyrUp(gaussPyrImg[i + 1], upsampleImage, gaussPyrImg[i].size());
		Mat currentImage = gaussPyrImg[i] - upsampleImage;
		//showImage("currentImage", currentImage);
		//cout << currentImage.size() << endl;
		_laplacianPyramid.push_back(currentImage);
	}
	return _laplacianPyramid;
}


vector> blendPyramid(vector> pyrA, vector> pyrB, vector> pyrMask)
{
	int levels = pyrA.size();
	vector> blendedPyramid;
	for (int i = 0; i < levels; i++)
	{
		Mat blendedImage = pyrA[i].mul(1.0 - pyrMask[levels -1 - i]) + pyrB[i].mul(pyrMask[levels - 1 - i]);
		//showImage("blendedImage", blendedImage);
		blendedPyramid.push_back(blendedImage);
	}
	return blendedPyramid;
}

Mat collapsePyramid(vector> blendedPyramid)
{
	int levels = blendedPyramid.size();
	Mat currentImage = blendedPyramid[0];
	for (int i = 1; i < levels; i++)
	{
		pyrUp(currentImage, currentImage, blendedPyramid[i].size());
		currentImage += blendedPyramid[i];
		//showImage("currentImage", currentImage);
	}
	Mat blendedImage;
	convertScaleAbs(currentImage, blendedImage, 255.0);
	return blendedImage;
}

int main() 
{
	// Read the images, assuming images to be of same size
	Mat A = imread("ghost rider.jpg", IMREAD_COLOR);
	Mat B = imread("nicholas cage.jpg", IMREAD_COLOR);
	//showImage("leftImage", A);
	//showImage("rightImage", B);
	
	if (A.size() != B.size())
	{
		A = Mat(A, Range::all(), Range(0, A.cols));
		B = Mat(B, Range(0, A.rows), Range(0, A.cols));
	}

	int height = A.rows;
	int width = A.cols;

	// Convert images to grayscale, float
	Mat _imgA, _imgB, imgA, imgB;
	cvtColor(A, _imgA, CV_BGR2GRAY);
	cvtColor(B, _imgB, CV_BGR2GRAY);

	_imgA.convertTo(imgA, CV_32F, 1.0 / 255.0);
	_imgB.convertTo(imgB, CV_32F, 1.0 / 255.0);


	// Create mask
	Mat_ mask(height, width, 0.0);
	mask(Range::all(), Range(mask.cols / 2 + 2, mask.cols )) = 1.0;
	//showImage("mask", mask * 255);

	// Create gaussian pyramids for both the images and the mask
	int levels = floor(log2(min(width, height)));

	vector> gaussPyrA = gaussianPyramid(imgA, levels);
	vector> gaussPyrB = gaussianPyramid(imgB, levels);
	vector> gaussPyrMask = gaussianPyramid(mask, levels);

	// Create Laplacian pyramids from both the images 
    vector> laplacePyrA = laplacianPyramid(gaussPyrA);
	vector> laplacePyrB = laplacianPyramid(gaussPyrB);

	// Blend laplacian pyramid
	vector> blendedPyr = blendPyramid(laplacePyrA, laplacePyrB, gaussPyrMask);

	// Reconstruct image from blended pyramid
	Mat blendedImg = collapsePyramid(blendedPyr);
	showImage("blendedImg", blendedImg);

	return 0;
}

效果如下：

做了简单修改使得C++代码可以处理彩色图片，代码基本重复就不贴了，可以在这里下载：

https://download.csdn.net/download/u014485485/11146735

效果：

 

 

[1]http://graphics.cs.cmu.edu/courses/15-463/2010_spring/Lectures/blending.pdf

[2] https://web.stanford.edu/class/cs231m/lectures/lecture-5-stitching-blending.pdf

[3] https://github.com/BigRedT/Image_Blending