基于小波变换的图像融合（附加视频融合）代码

一、小波变换

小波分析是一个比较难的分支，用户采用小波变换，可以实现图像压缩，振动信号的分解与重构等，因此在实际工程上应用较广泛。小波分析与Fourier变换相比，小波变换是空间域和频率域的局部变换，因而能有效地从信号中提取信息。小波变换通过伸缩和平移等基本运算，实现对信号的多尺度分解与重构，从而很大程度上解决了Fourier变换带来的很多难题。
小波分析作一个新的数学分支，它是泛函分析、Fourier分析、数值分析的完美结晶；小波分析也是一种“时间—尺度”分析和多分辨分析的新技术，它在信号分析、语音合成、图像压缩与识别、大气与海洋波分析等方面的研究，都有广泛的应用。

二、图像融合

1、采用对比度拉伸，完成红外图像增强

# 裁剪线性RGB对比度拉伸：（去掉2%百分位以下的数，去掉98%百分位以上的数，上下百分位数一般相同，并设置输出上下限）
def truncated_linear_stretch(image, truncated_value=2, maxout=255, min_out=0):
    def gray_process(gray, maxout=maxout, minout=min_out):
        truncated_down = np.percentile(gray, truncated_value)
        truncated_up = np.percentile(gray, 100 - truncated_value)
        gray_new = ((maxout - minout) / (truncated_up - truncated_down)) * gray
        gray_new[gray_new < minout] = minout
        gray_new[gray_new > maxout] = maxout
        return np.uint8(gray_new)
    (b, g, r) = cv2.split(image)
    b = gray_process(b)
    g = gray_process(g)
    r = gray_process(r)
    result = cv2.merge((b, g, r))# 合并每一个通道
    return result

2、采用Surf特征点匹配，完成图像配准

# RGB图片配准函数，采用白天的可见光与红外灰度图，计算两者Surf共同特征点，之间的仿射矩阵。
def Images_matching(img_base, img_target):
    img_base=cv2.cvtColor(img_base, cv2.COLOR_BGR2GRAY)
    img_target=cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY)
    hessian = 400
    # 初始化surf算子
    surf = cv2.xfeatures2d.SURF_create(hessian)
    # surf = cv2.xfeatures2d_SURF.create(hessian)
    # surf = cv2.SIFT_create(hessian)
    # 使用surf算子计算特征点和特征点周围的特征向量
    kp1, des1 = surf.detectAndCompute(img_base, None)  # 1136    1136, 64
    kp2, des2 = surf.detectAndCompute(img_target, None)
    # 进行KNN特征匹配
    FLANN_INDEX_KDTREE = 0  # 建立FLANN匹配器的参数
    indexParams = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)  # 配置索引，密度树的数量为5
    searchParams = dict(checks=50)  # 指定递归次数
    flann = cv2.FlannBasedMatcher(indexParams, searchParams)  # 建立匹配器
    matches = flann.knnMatch(des1, des2, k=2)  # 得出匹配的关键点  list: 1136
    good = []
    # 提取优秀的特征点
    for m, n in matches:
        if m.distance < 0.7 * n.distance:  # 如果第一个邻近距离比第二个邻近距离的0.7倍小，则保留
            good.append(m)   # 134
    src_pts = np.array([kp1[m.queryIdx].pt for m in good])  # 查询图像的特征描述子索引  # 134, 2
    dst_pts = np.array([kp2[m.trainIdx].pt for m in good])  # 训练(模板)图像的特征描述子索引
    H = cv2.findHomography(dst_pts, src_pts, cv2.RANSAC)  # 生成变换矩阵  H[0]: 3, 3  H[1]: 134, 1
    return H[0]

3、采用HSV通道小波变换，完成图像融合

# YUV通道小波变换RGB融合，低频均值高频最大值
def Images_fusion(img_base,img_target):          # 可见光，配准后的红外光
    # 按照不同的融合方式，计算通道融合
    def fuseCoeff(cooef1, cooef2, method):
        if (method == 'mean'):
            cooef = (abs(cooef1) + abs(cooef2)) / 2  # abs 绝对值
            # cooef=0.5*cooef1+1.5*cooef2
        elif (method == 'min'):
            cooef = np.minimum(cooef1, cooef2)
        elif (method == 'max'):
            cooef = np.maximum(abs(cooef1), abs(cooef2))
        return cooef
    # HSV的小波融合过程
    LOW_METHOD = 'mean'
    HIGH_METHOD = 'max'
    YUVimg_base = cv2.cvtColor(img_base, cv2.COLOR_BGR2YUV)  # 可见光转换为HSV
    # cv2.imwrite("D:/VS/vsprj/cuda/cudawavetest/2.jpg", YUVimg_base)
    grayimg_target = cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY)  #红外灰度化
    # cv2.imwrite("D:/VS/vsprj/cuda/cudawavetest/1.jpg",grayimg_target)
    Yimg_base = YUVimg_base[:, :, 0]  # 1024，1024
    wavelet = 'haar'
    cooef_base = pywt.wavedec2(Yimg_base[:, :], wavelet, level=1)  # base灰度图的小波展开  512, 512    3
    start = time.time()
    cooef_target = pywt.wavedec2(grayimg_target[:, :], wavelet, level=1)  # target灰度图的小波展开   512, 512    3
    end = time.time()-start
    print("小波展开:{}".format(end))
    fusedCooef = []
    for i in range(len(cooef_base)):
        if (i == 0):
            fusedCooef.append(fuseCoeff(cooef_base[0], cooef_target[0], LOW_METHOD))  # 低频部分取均值
        else:
            # 高频部分取最大值
            c1 = fuseCoeff(cooef_base[i][0], cooef_target[i][0], HIGH_METHOD)
            c2 = fuseCoeff(cooef_base[i][1], cooef_target[i][1], HIGH_METHOD)
            c3 = fuseCoeff(cooef_base[i][2], cooef_target[i][2], HIGH_METHOD)
            fusedCooef.append((c1, c2, c3))  # 高频合并
    tempfusedImage = pywt.waverec2(fusedCooef, wavelet)  # 小波逆变换
    fusedImage = np.multiply(np.divide(tempfusedImage - np.min(tempfusedImage), (np.max(tempfusedImage) -\
                            np.min(tempfusedImage))), 255)  # 逆变换后归一至（0，255）
    start = time.time()
    fusedImage = fusedImage.astype(np.uint8)
    Yimg_new = fusedImage
    fusedYUV = cv2.merge([Yimg_new, YUVimg_base[:, :, 1], YUVimg_base[:, :, 2]])  # 用小波变换替换V通道
    end = time.time() - start
    print("图像重建:{}".format(end))
    fusedBGR = cv2.cvtColor(fusedYUV, cv2.COLOR_YUV2BGR)  # 融合后的HSV转为BGR
    return fusedBGR

4、总的代码

import cv2
import numpy as np
import pywt #引入小波模块
import time
from PIL import Image

# 裁剪线性RGB对比度拉伸：（去掉2%百分位以下的数，去掉98%百分位以上的数，上下百分位数一般相同，并设置输出上下限）
def truncated_linear_stretch(image, truncated_value=2, maxout=255, min_out=0):
    def gray_process(gray, maxout=maxout, minout=min_out):
        truncated_down = np.percentile(gray, truncated_value)
        truncated_up = np.percentile(gray, 100 - truncated_value)
        gray_new = ((maxout - minout) / (truncated_up - truncated_down)) * gray
        gray_new[gray_new < minout] = minout
        gray_new[gray_new > maxout] = maxout
        return np.uint8(gray_new)
    (b, g, r) = cv2.split(image)
    b = gray_process(b)
    g = gray_process(g)
    r = gray_process(r)
    result = cv2.merge((b, g, r))# 合并每一个通道
    return result

# RGB图片配准函数，采用白天的可见光与红外灰度图，计算两者Surf共同特征点，之间的仿射矩阵。
def Images_matching(img_base, img_target):
    img_base=cv2.cvtColor(img_base, cv2.COLOR_BGR2GRAY)
    img_target=cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY)
    hessian = 400
    # 初始化surf算子
    surf = cv2.xfeatures2d.SURF_create(hessian)
    # surf = cv2.xfeatures2d_SURF.create(hessian)
    # surf = cv2.SIFT_create(hessian)
    # 使用surf算子计算特征点和特征点周围的特征向量
    kp1, des1 = surf.detectAndCompute(img_base, None)  # 1136    1136, 64
    kp2, des2 = surf.detectAndCompute(img_target, None)
    # 进行KNN特征匹配
    FLANN_INDEX_KDTREE = 0  # 建立FLANN匹配器的参数
    indexParams = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)  # 配置索引，密度树的数量为5
    searchParams = dict(checks=50)  # 指定递归次数
    flann = cv2.FlannBasedMatcher(indexParams, searchParams)  # 建立匹配器
    matches = flann.knnMatch(des1, des2, k=2)  # 得出匹配的关键点  list: 1136
    good = []
    # 提取优秀的特征点
    for m, n in matches:
        if m.distance < 0.7 * n.distance:  # 如果第一个邻近距离比第二个邻近距离的0.7倍小，则保留
            good.append(m)   # 134
    src_pts = np.array([kp1[m.queryIdx].pt for m in good])  # 查询图像的特征描述子索引  # 134, 2
    dst_pts = np.array([kp2[m.trainIdx].pt for m in good])  # 训练(模板)图像的特征描述子索引
    H = cv2.findHomography(dst_pts, src_pts, cv2.RANSAC)  # 生成变换矩阵  H[0]: 3, 3  H[1]: 134, 1
    return H[0]

# YUV通道小波变换RGB融合，低频均值高频最大值
def Images_fusion(img_base,img_target):          # 可见光，配准后的红外光
    # 按照不同的融合方式，计算通道融合
    def fuseCoeff(cooef1, cooef2, method):
        if (method == 'mean'):
            cooef = (abs(cooef1) + abs(cooef2)) / 2  # abs 绝对值
            # cooef=0.5*cooef1+1.5*cooef2
        elif (method == 'min'):
            cooef = np.minimum(cooef1, cooef2)
        elif (method == 'max'):
            cooef = np.maximum(abs(cooef1), abs(cooef2))
        return cooef
    # HSV的小波融合过程
    LOW_METHOD = 'mean'
    HIGH_METHOD = 'max'
    YUVimg_base = cv2.cvtColor(img_base, cv2.COLOR_BGR2YUV)  # 可见光转换为HSV
    # cv2.imwrite("D:/VS/vsprj/cuda/cudawavetest/2.jpg", YUVimg_base)
    grayimg_target = cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY)  #红外灰度化
    # cv2.imwrite("D:/VS/vsprj/cuda/cudawavetest/1.jpg",grayimg_target)
    Yimg_base = YUVimg_base[:, :, 0]  # 1024，1024
    wavelet = 'haar'
    cooef_base = pywt.wavedec2(Yimg_base[:, :], wavelet, level=1)  # base灰度图的小波展开  512, 512    3
    start = time.time()
    cooef_target = pywt.wavedec2(grayimg_target[:, :], wavelet, level=1)  # target灰度图的小波展开   512, 512    3
    end = time.time()-start
    print("小波展开:{}".format(end))
    fusedCooef = []
    for i in range(len(cooef_base)):
        if (i == 0):
            fusedCooef.append(fuseCoeff(cooef_base[0], cooef_target[0], LOW_METHOD))  # 低频部分取均值
        else:
            # 高频部分取最大值
            c1 = fuseCoeff(cooef_base[i][0], cooef_target[i][0], HIGH_METHOD)
            c2 = fuseCoeff(cooef_base[i][1], cooef_target[i][1], HIGH_METHOD)
            c3 = fuseCoeff(cooef_base[i][2], cooef_target[i][2], HIGH_METHOD)
            fusedCooef.append((c1, c2, c3))  # 高频合并
    tempfusedImage = pywt.waverec2(fusedCooef, wavelet)  # 小波逆变换
    fusedImage = np.multiply(np.divide(tempfusedImage - np.min(tempfusedImage), (np.max(tempfusedImage) -\
                            np.min(tempfusedImage))), 255)  # 逆变换后归一至（0，255）
    start = time.time()
    fusedImage = fusedImage.astype(np.uint8)
    Yimg_new = fusedImage
    fusedYUV = cv2.merge([Yimg_new, YUVimg_base[:, :, 1], YUVimg_base[:, :, 2]])  # 用小波变换替换V通道
    end = time.time() - start
    print("图像重建:{}".format(end))
    fusedBGR = cv2.cvtColor(fusedYUV, cv2.COLOR_YUV2BGR)  # 融合后的HSV转为BGR
    return fusedBGR
#
def main():
    matchimg_di = cv2.imread('images/oripics/basic_day_infrared.jpg')  # 1080, 1920, 3
    matchimg_dv = cv2.imread('images/oripics/basic_day_visual.jpg')
    # matchimg_di = cv2.imread('left_2020_11_30-15_15_31.jpg')  # 1080, 1920, 3
    # matchimg_dv = cv2.imread('right_2020_11_30-15_15_31.jpg')
    # 1080, 1920, 3
    orimg_nv=matchimg_dv
    orimg_ni=matchimg_di
    # orimg_nv = cv2.imread('images/oripics/night_visual_40m.jpg')
    # orimg_ni = cv2.imread('images/oripics/night_infrared_40m.jpg')
    # orimg_nv = cv2.imread('images/oripics/left_2020_11_30-20_06_54.jpg')
    # orimg_ni = cv2.imread('images/oripics/right_2020_11_30-20_06_54.jpg')
    # orimg_nv = cv2.imread('images/oripics/left_2020_11_30-20_08_09.jpg')
    # orimg_ni = cv2.imread('images/oripics/right_2020_11_30-20_08_09.jpg')
    orimg_nv = cv2.imread('images/oripics/left_video_2020_11_30-20_09_32_73.jpg')  # 1024, 1024, 3
    orimg_ni = cv2.imread('images/oripics/right_video_2020_11_30-20_09_32_73.jpg')  # 1024, 1024, 3

    #用白天图像进行配准
    # enhance_matchimg_di = truncated_linear_stretch(matchimg_di)# 配准模板红外图像RGB增强
    h, w = orimg_nv.shape[:2]   # 1024 1024
    H = Images_matching(matchimg_dv, matchimg_di)   # (3, 3)

    # enhance_orimg_ni=truncated_linear_stretch(orimg_ni) # 需融合红外图像RGB增强
    matched_ni = cv2.warpPerspective(orimg_ni, H, (w, h))
    cv2.imwrite("./1.jpg",matched_ni)# 红外图像按照仿射矩阵配准 1024, 1024, 3
    start = time.time()
    fusion = Images_fusion(orimg_nv, matched_ni)
    cv2.imwrite("./2.jpg",fusion)
    end = time.time()-start
    print(end)
    # enhance=truncated_linear_stretch(fusion)#融合图像RGB增强

    # cv2.imshow('0', cv2.resize(orimg_nv, (600, 400)))
    # cv2.imshow('1', cv2.resize(orimg_ni, (600, 400)))
    cv2.imshow('2', cv2.resize(fusion, (1200, 800)))
    cv2.waitKey(0)

    # name_fusionfile='YUV_04.jpg'
    # path_fusionfile='images/fusion/'+name_fusionfile
    # cv2.imwrite(path_fusionfile, fusion)
if __name__ == '__main__':
    main()    

5、融合效果 上图

融合图片


融合后，效果一般，但是将就看吧，熟悉一下融合过程。

三、视频融合

一个可见光摄像头，一个红外摄像头，就可以实现实时夜间车外图像融合啦

video_path_infrared = "../videos/ir/video_2020_11_30-20_05_30.avi"
    video_path_visible = "../videos/vi/video_2020_11_30-20_05_30.avi"
    video_save_path = "../videos/out.avi"
    video_fps = 25

    # matchimg_di = cv2.imread('images/oripics/basic_day_infrared.jpg')  # 1080, 1920, 3
    # matchimg_dv = cv2.imread('images/oripics/basic_day_visual.jpg')
    matchimg_di = cv2.imread('left_2020_11_30-15_15_31.jpg')  # 1080, 1920, 3
    matchimg_dv = cv2.imread('right_2020_11_30-15_15_31.jpg')
    H = Images_matching(matchimg_dv, matchimg_di)

    capture_in = cv2.VideoCapture(video_path_infrared)
    capture_vi = cv2.VideoCapture(video_path_visible)
    if video_save_path != "":
        fourcc = cv2.VideoWriter_fourcc(*'XVID')
        size = (int(capture_vi.get(cv2.CAP_PROP_FRAME_WIDTH)), int(capture_vi.get(cv2.CAP_PROP_FRAME_HEIGHT)))
        out = cv2.VideoWriter(video_save_path, fourcc, video_fps, size)

    fps = 0.0
    while (True):
        t1 = time.time()
        # 读取某一帧
        ref1, frame1 = capture_in.read()
        ref2, frame2 = capture_vi.read()
        # 格式转变，BGRtoRGB
        frame1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB)
        frame2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2RGB)
        # 转变成Image
        # frame1 = Image.fromarray(np.uint8(frame1))
        # frame2 = Image.fromarray(np.uint8(frame2))
        h, w = frame1.shape[:2]
        matched_ni = cv2.warpPerspective(frame2, H, (w, h))
        cv2.imwrite("./1.jpg", matched_ni)
        # 进行检测
        fusion = Images_fusion(frame1, matched_ni)
        fusion = np.array(fusion)
        end = time.time() - t1
        # RGBtoBGR满足opencv显示格式
        frame = cv2.cvtColor(fusion, cv2.COLOR_RGB2BGR)

        fps = (fps + (1. / end )) / 2
        print("fps= %.2f" % (fps))
        frame = cv2.putText(frame, "fps= %.2f" % (fps), (0, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)

        cv2.imshow("video", frame)
        c = cv2.waitKey(1) & 0xff
        if video_save_path != "":
            out.write(frame)

        if c == 27:
            capture.release()
            break
    capture.release()
    out.release()
    cv2.destroyAllWindows()

一定选对配准图片，不然会出现鬼影O

四、总结

这个方法试出来视频融合的时候如何加速，大家如果有什么好方法，多教教我！