【OpenCV】使用特征点匹配完成数字视频稳像

环境：OpenCV-Python 4.1.1

           Python 3.6

---

主要有三个关键点：

1. 运动估计

2. 运动平滑

3. 图像合成

首先是遍历视频文件的所有帧，寻找每一帧图像中好的跟踪点，OpenCV已经提供了函数goodFeaturesToTrack来获取这些特征点。获得当前帧的特征点之后，就可以使用calcOpticalFlowPyrLK跟踪这些特征点了。

cap = cv2.VideoCapture('test.mp4')
n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = cap.get(cv2.CAP_PROP_FPS)
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter('out.avi', fourcc, fps, (2 * w, h))

_, prev = cap.read()
prev_gray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)

transforms = np.zeros((n_frames-1, 3), np.float32)

for i in range(n_frames-2):
    prev_pts = cv2.goodFeaturesToTrack(prev_gray,
                                       maxCorners=200,
                                       qualityLevel=0.01,
                                       minDistance=30,
                                       blockSize=3)
    success, curr = cap.read()
    if not success:
        break

    curr_gray = cv2.cvtColor(curr, cv2.COLOR_BGR2GRAY)
    curr_pts, status, err = cv2.calcOpticalFlowPyrLK(prev_gray, curr_gray, prev_pts, None)
    assert prev_pts.shape == curr_pts.shape

    #把跟丢的点去掉
    idx = np.where(status == 1)[0]
    prev_pts = prev_pts[idx]
    curr_pts = curr_pts[idx]

接下来就是运动估计。我们已经知道了特征点在前一帧图像中的位置，并且通过跟踪得到了特征点在当前帧的位置，那么就可以得到前一帧到当前帧的欧几里得变换。

    m, _ = cv2.estimateAffinePartial2D(prev_pts, curr_pts)
    dx = m[0, 2]
    dy = m[1, 2]
    da = np.arctan2(m[1, 0], m[0, 0])
    transforms[i] = [dx, dy, da]

    prev_gray = curr_gray

为了得到更稳定的结果，将视频的所有运动进行平滑。

SMOOTHING_RADIUS = 50

def movingAverage(curve, radius):
    window_size = 2 * radius + 1
    f = np.ones(window_size)/window_size
    curve_pad = np.lib.pad(curve, (radius, radius), 'edge')
    curve_smoothed = np.convolve(curve_pad, f, mode='same')
    curve_smoothed = curve_smoothed[radius:-radius]
    return curve_smoothed

def smooth(trajectory):
    smoothed_trajectory = np.copy(trajectory)
    for i in range(3):
        smoothed_trajectory[:, i] = movingAverage(trajectory[:, i], radius=SMOOTHING_RADIUS)
    return  smoothed_trajectory

trajectory = np.cumsum(transforms, axis=0)
smoothed_trajectory = smooth(trajectory)
difference = smoothed_trajectory - trajectory
transforms_smooth = transforms + difference

接下来就可以将平滑后的运动应用到视频帧。

#为了缓和变换之后图像的黑边，这里简单的由中心放大4%
def fixBorder(frame):
    s = frame.shape
    T = cv2.getRotationMatrix2D((s[1]/2, s[0]/2), 0, 1.04)
    frame = cv2.warpAffine(frame, T, (s[1], s[0]))
    return frame

cap.set(cv2.CAP_PROP_POS_FRAMES, 0)

for i in range(n_frames-2):
    success, frame = cap.read()
    if not success:
        break

    dx = transforms_smooth[i, 0]
    dy = transforms_smooth[i, 1]
    da = transforms_smooth[i, 2]

    m = np.zeros((2, 3), np.float32)
    m[0, 0] = np.cos(da)
    m[0, 1] = -np.sin(da)
    m[1, 0] = np.sin(da)
    m[1, 1] = np.cos(da)
    m[0, 2] = dx
    m[1, 2] = dy

    frame_stabilized = cv2.warpAffine(frame, m, (w, h))
    frame_stabilized = fixBorder(frame_stabilized)
    frame_out = cv2.hconcat([frame, frame_stabilized])
    if(frame_out.shape[1] > 1920):
        frame_out = cv2.resize(frame_out, (frame_out.shape[1]/2, frame_out.shape[0]/2))

    cv2.imshow("Before VS. After", frame_out)
    cv2.waitKey(10)
    out.write(frame_out)

cap.release()
out.release()
cv2.destroyAllWindows()