LK光流对两张图配准

从构建非线性目标函数的角度出发，光流三大假设，分别是亮度一致性，时域一致性和空域一致性

其中，亮度一致性构建了光流方程，时域一致性构建超定方程求解方式，而空域一致性，就是所谓的两帧之间运动微小假设，则引出了泰勒公式+梯度下降的方式(牛顿高斯迭代法)来求解非线性目标函数。

 

根据目标函数求导，推导可得：

 光流算法，输入是ref帧图像的特征点坐标；输出是tar帧图像的特征点坐标。在此处还需要设置一些额外的参数，如滑动窗口的大小(win_size)，迭代次数(max_iter)

import numpy as np
from menpo import image as mpimg
from menpo.shape import PointDirectedGraph
from menpofit.lk import LucasKanadeFitter, GradientCorrelation, GradientImages, ECC, SSD
from menpofit.transform import DifferentiableAlignmentSimilarity, DifferentiableAlignmentAffine
from matplotlib.patches import Polygon
import pylab as plt
import cv2
from PIL import Image
import os.path as osp
import os
from tqdm import tqdm 
from skimage.registration import optical_flow_tvl1, optical_flow_ilk
from skimage.transform import warp

def menpofit_lk(moving_img, fixed_img, init_bbox, scales=[1/16, 1/8, 1/4, 1/2, 1], tform_tp='similarity', residual_cls=SSD):

    np_init_bbox_yx = np.array(init_bbox)[:, ::-1]
    
    np_moving_img = np.array(Image.open(moving_img).convert('L')).astype('f4') / 255.0
    np_fixed_img = np.array(Image.open(fixed_img).convert('L').convert('L')).astype('f4') / 255.0

    moving_img = mpimg.Image(np_moving_img[None])
    fixed_img = mpimg.Image(np_fixed_img[None])
    h, w = moving_img.shape

    bbox_edges = [[0,1], [1,2], [2,3], [3,0]]

    moving_img.landmarks['bounding_box'] = PointDirectedGraph.init_from_edges(
    points=np.array([
    [0, 0],
    [h-1, 0],
    [h-1, w-1],
    [0, w-1],
    ]),
    edges=bbox_edges
    )

    init_bb = PointDirectedGraph.init_from_edges(
    points=np_init_bbox_yx,
    edges=bbox_edges
    )

    if tform_tp == 'similarity':
        transform = DifferentiableAlignmentSimilarity
    else:
        transform = DifferentiableAlignmentAffine

    fitter = LucasKanadeFitter(moving_img, scales=scales, residual_cls=residual_cls, transform=transform)
    fr = fitter.fit_from_bb(fixed_img, init_bb)
    refined_bbox = fr.final_shape.points[:, ::-1]
    

    return refined_bbox

if __name__ == '__main__':
    init_bbox = [[0,628],[628,628],[628,0],[0,0]]
    ### test_aug_to_temp就是test_aug图片地址对应的temp图片地址
    for test_aug in test_aug_to_temp:
        temp_path = test_aug_to_temp[test_aug]
        refined_bbox = menpofit_lk(test_aug, temp_path, init_bbox, scales=[1/16, 1/8, 1/4, 1/2, 1], tform_tp='similarity', residual_cls=SSD)
        ransacReprojThreshold = 4
        H, status =cv2.findHomography(np.array(refined_bbox),np.array(init_bbox),cv2.RANSAC,ransacReprojThreshold);
        print(H)