基于邻域嵌入的超分辨率方法理解与实现
声明:
本文为博主原创，只是个人理解，如有理解错误的地方，欢迎指正。代码部分有参考导师提供的部分代码，来源不详。
参考文献:
Super-Resolution through Neighbor Embedding
(http://doi.ieeecomputersociety.org/10.1109/CVPR.2004.243)
算法描述:
给定一副低分辨率图像（LR）和多幅训练图像（SR training images），通过流型学习，得到目标高分辨率图像（SR）。
详细步骤：
- 降质
对训练图像降质（比如目标SR是LR的N倍大，则把训练图像缩小N倍）得到LR training images； - 分块
把LR和LR training images分成大小s*s ，重叠区域宽为overlap的重叠图像块（image pitch），SR training images则是Ns*Ns大的重叠图像块（image pitch），其中low-resolution training images中的每一块对应SR training images中每一块； - 特征提取
计算LR和LR training images各像素点的一阶梯度和二阶梯度，用一个特征向量代表一个图像块，特征向量的集合分别为XT，XS。对于SR training images，只需要每个pitch内每个像素减去块内所有像素值的平均值，这里为什么减去均值的原因是，超分辨率的本质是要找回由于图像降质过程丢失的高频细节信息，高频细节信息是由低分辨率图像中的中频信息所决定，而不是整个低分辨率图像。减去它的均值其实就是中频信息。SR training pitch同样是用一个向量表示，集合记为YS； - NE
对于XT中的每一个列向量，在XS中找到K个与之最相似（欧式距离最近）的列向量（neighbor），求出每个neighbor的权值，然后除以所有权值之和，使这K个权值相加为一（归一化）；
前面提到每个LR training pitch对应一个SR training pitch，即用前面选出的K个neighbor对应的SR training pitch分别乘以对应的权重，加起来即为重建的image pitch的特征向量。
重复步骤4，直到所有的XT中向量都处理完，此时得到一个重叠的image pitch集合，记为YT； - 图像重建
现在已知YT，只要把YT中的特征向量恢复到图像即可。只需按顺序把YT中的每个向量代表的图像块恢复到原来的位置，重叠区域值为相邻image pitch在该区域的平均值表示，最后加上LR每个图像块的中频信息，即SR每个图像块像素值减去平均值，即可求出目标SR。
代码实现 ：
MATLAB语言：
%——————————————————————————————————————————————————————————
%主程序
function example
s = 3; %3*3的图像块
overlap = 2; %重叠区域宽为2
K = 5; %在XS中查找K个neighbor
for i = 1:X %X幅训练图像
RGB_YS = imread('.\pic\....bmp','bmp');%输入训练图像的地址
RGB_XS = downsize(RGB_YS,4); %假设放大4倍
YIQ = rgb2ntsc(RGB_XS);XS = YIQ(:,:,1);
%这里是把RGB图像转换为YIQ图像,然后针对Y放大,具体转换原因文献里
%有提到,这里就不赘述了。
YIQ = rgb2ntsc(RGB_YS);YS = YIQ(:,:,1);
[XSp,XSrow,XScol] = overlapcut(XS, s, overlap);
[YSp,YSrow,YScol] = overlapcut(YS, 4*s, 4*overlap);
XSv2 = findgradient(XSp,XS,XSrow,XScol);
%计算每个像素值的梯度值
YSv2 = findfeature(YSp,YSrow,YScol);
%计算YS的特征向量
if i==1
XSv = XSv2;
YSv = YSv2;
else
XSv(:,size(XSv,2)+1:size(XSv,2)+size(XSv2,2)) = XSv2;
YSv(:,size(YSv,2)+1:size(YSv,2)+size(YSv2,2)) = YSv2;
%添加到XS,YS集合中
end
end
RGB_XT = imread('.\pic\low.bmp','bmp');
YIQ = rgb2ntsc(RGB_XT); XT = YIQ(:,:,1);
YIQ = rgb2ntsc(upsize(RGB_XT,4));IQ_YT = YIQ(:,:,2:3);
%save IQ,为了后面重建的时候恢复成rgb图像
[XTp,XTrow,XTcol] = overlapcut(XT, 3, 2);
XTmean = findmean(XTp,XTrow,XTcol);%提取中频信息
XTv = findgradient(XTp,XT,XTrow,XTcol);
YTv = naneighbor(XTv,XSv,YSv,K);%计算出YT的特征向量集合
ET = findimage1(YTv,XTrow,XTcol,XTmean,IQ_YT,12,8); %重建
RGB_ET = ntsc2rgb(ET);%YIQ恢复到RGB图像
figure;image(RGB_XT);axis off;title('input low-res');
figure;image(RGB_ET);axis off;title('output super-res');
%——————————————————————————————————————————————————————————
%降质
function B = downsize(A,ratio)
[m,n,~] = size(A);
newm = floor(m/ratio);
newn = floor(n/ratio);
A = double(A);
B = zeros(newm,newn,3);
for i=1:ratio:m
minx = min(i+ratio-1,m);
for j=1:ratio:n
miny = min(j+ratio-1,n);
temp = round(sum(sum(A(i:minx,j:miny,:),2))/((minx-i+1)*(miny-j+1)));
ii = (i-1)/ratio+1;
jj=(j-1)/ratio+1;
B(ii,jj,:) = temp;
if miny == n
break;
end
end
if minx == m
break;
end
end
B = uint8(B);
%——————————————————————————————————————————————————————————
%分块
function [B,row,col] = overlapcut(A,s,overlap)
[m,n] = size(A);
row = ceil((m-overlap)/(s-overlap));
col = ceil((n-overlap)/(s-overlap));
B = zeros(s,s,row*col);
for i=1:row
for j=1:col
index = (i-1)*col + j;
ii = min((i-1)*(s-overlap)+s,m)-(i-1)*(s-overlap);
jj = min((j-1)*(s-overlap)+s,n)-(j-1)*(s-overlap);
B(1:ii,1:jj,index) = A((i-1)*(s-overlap)+1:min((i-1)*(s-overlap)+s,m),(j-1)*(s-overlap)+1:min((j-1)*(s-overlap)+s,n));
end
end
%——————————————————————————————————————————————————————————
%SR training image特征提取
function f = findfeature(A,row,col)
[~,s,n] = size(A); %n is # of patches
if (row*col~=n)
fprintf('input parameter error!');
return;
end
f = zeros(s*s,n);
for i=1:n
temp = A(:,:,i);
vector = reshape(temp',s*s,1);
vector = vector - mean(vector);
f(:,i) = vector; %all piexl values (-mean) in the patches
end
%——————————————————————————————————————————————————————————
%LR和LR training image 的梯度值
function f = findgradient(A,B,row,col)
component of some image
[~,s,n] = size(A); %n is # of patches
[ii,jj] = size(B);
if (row*col~=n)
fprintf('input parameter error!');
return;
end
grad = zeros(ii,jj,4);
for i=1:ii %for each pixel in B matrix
for j=1:jj
if (j+1<=jj) && (j-1>=1)
grad(i,j,1) = B(i,j+1)-B(i,j-1);
end
if (i+1<=ii) && (i-1>=1)
grad(i,j,2) = B(i+1,j)-B(i-1,j);
end
if (i+2<=ii) && (i-2>=1)
grad(i,j,3) = B(i+2,j)-2*B(i,j)+B(i-2,j);
end
if (j+2<=jj) && (j-2>=1)
grad(i,j,4) = B(i,j+2)-2*B(i,j)+B(i,j-2);
end
end
end
f = zeros(4*s*s,n);
for p=1:n
i = ceil(p/col); % left top position of p-th patch
j = mod(p-1,col)+1; % in grad matrix is (i,j)
add = 0;
for si=0:s-1
for sj=0:s-1
for num=1:size(grad,3)
add = add+1;
f(add,p) = grad(i+si,j+sj,num);
end
end
end
end
%——————————————————————————————————————————————————————————
%NE算法
function [YT,U,neighborhood] = naneighbor(XT,XS,YS,K)
[~,T] = size(XT);
[M,~] = size(YS);
neighbors\n',K);
neighborhood = zeros(K,T);
for i = 1:T
temp = XT(:,i);
distance = dist2(temp',XS');
[~,index] = sort(distance');
neighborhood(:,i) = index(2:(K+1));
end
tol=1e-4; % regularlizer in case constrained fits are ill conditioned
U = zeros(K,T);
for ii=1:T
z = XS(:,neighborhood(:,ii))-repmat(XT(:,ii),1,K); % shift ith pt to origin
C = z'*z; % local covariance
if trace(C)==0
C = C + eye(K,K)*tol; % regularlization
else
C = C + eye(K,K)*tol*trace(C);
end
U(:,ii) = C\ones(K,1); % solve C*u=1
U(:,ii) = U(:,ii)/sum(U(:,ii)); % enforce sum(u)=1
end;
YT = zeros(M,T);
for ii = 1:T
YT(:,ii) = YS(:,neighborhood(:,ii))*U(:,ii);
end
%——————————————————————————————————————————————————————————
%提取中频信息
function XTmean = findmean(XTp,row,col)
[~,~,n] = size(XTp);
if (n~=row*col)
fprintf('input parameter error!')
return;
end
XTmean = zeros(row,col);
for i=1:row
for j=1:col
index = (i-1)*col+j;
XTmean(i,j) = mean(mean(XTp(:,:,index)));
end
end
%——————————————————————————————————————————————————————————
%重建图像
function YT = findimage1(YTv,prow,pcol,XTmean,IQ,s,overlap)
[row,col,~] = size(IQ);
[~,N] = size(YTv);
if (N~=prow*pcol)
fprintf('input parameter error!');
return;
end
Y = zeros(prow*(s-overlap)+overlap,pcol*(s-overlap)+overlap);
flag = zeros(prow*(s-overlap)+overlap,pcol*(s-overlap)+overlap);
for i=1:prow
for j=1:pcol
for t=1:s
Y((s-overlap)*(i-1)+t,(s-overlap)*(j-1)+1:(s-overlap)*(j-1)+s) = XTmean(i,j) + Y((s-overlap)*(i-1)+t,(s-overlap)*(j-1)+1:(s-overlap)*(j-1)+s)+ YTv(s*(t-1)+1:s*t,(i-1)*pcol+j)';
flag((s-overlap)*(i-1)+t,(s-overlap)*(j-1)+1:(s-overlap)*(j-1)+s)= flag((s-overlap)*(i-1)+t,(s-overlap)*(j-1)+1:(s-overlap)*(j-1)+s) +1;
end
end
end
Y = Y(1:row,1:col);
flag = flag(1:row,1:col);
YT = zeros(row,col,3);
YT(:,:,1) = Y./flag;
YT(:,:,[2,3]) = IQ(:,:,:);
fprintf('Done.\n');
%——————————————————————————————————————————————————————————