ffmpeg 双路输入解析——以vf_overlay为例

ffmpeg里面处理两路输入的模块都放在libavfilter/dualinput.c里面
首先定义了一个context用于描述dualinput的信息

FFDualInputContext

typedef struct FFDualInputContext {
    FFFrameSync fs;

    AVFrame *(*process)(AVFilterContext *ctx, AVFrame *main, const AVFrame *second);
    int shortest;               ///< terminate stream when the second input terminates
    int repeatlast;             ///< repeat last second frame
    int skip_initial_unpaired;  ///< Skip initial frames that do not have a 2nd input
} FFDualInputContext;

process是一个函数指针，以vf_overlay为例，调用过程为

static AVFrame *do_blend(AVFilterContext *ctx, AVFrame *mainpic,
                         const AVFrame *second)
{
    ......
}

static av_cold void uninit(AVFilterContext *ctx)
{
    OverlayContext *s = ctx->priv;
    ff_dualinput_uninit(&s->dinput);
    ......
}

 s->dinput.process = do_blend;

但是到这里还是不明白main和second怎么传入do_blend的
用gdb分析，先b do_blend再bt，得到的结果为

调用过程backtrace

(gdb) bt 
#0  do_blend (ctx=0x2b6d7a0, mainpic=0x2b6cb80, second=0x2fa7e00) at libavfilter/vf_overla
y.c:772
#1  0x000000000066e3dc in process_frame (fs=0x2e60708) at libavfilter/dualinput.c:37
#2  0x000000000046fe07 in ff_framesync_process_frame (fs=0x2e60708, all=0) at libavfilter/
framesync.c:291
#3  0x000000000046fee5 in ff_framesync_filter_frame (fs=0x2e60708, inlink=0x2e61f00, in=0x
2fa7e00) at libavfilter/framesync.c:312
#4  0x000000000066e5c3 in ff_dualinput_filter_frame (s=0x2e60708, inlink=0x2e61f00, in=0x2
fa7e00) at libavfilter/dualinput.c:79
#5  0x0000000000541d9e in filter_frame (inlink=0x2e61f00, inpicref=0x2fa7e00) at libavfilt
er/vf_overlay.c:805
#6  0x0000000000458992 in ff_filter_frame_framed (link=0x2e61f00, frame=0x2fa7e00) at liba
vfilter/avfilter.c:1116
#7  0x0000000000458ff8 in ff_filter_frame_to_filter (link=0x2e61f00) at libavfilter/avfilt
er.c:1264
#8  0x00000000004591f0 in ff_filter_activate_default (filter=0x2b6d7a0) at libavfilter/avf
ilter.c:1315
#9  0x0000000000459350 in ff_filter_activate (filter=0x2b6d7a0) at libavfilter/avfilter.c:
1476
#10 0x000000000045db83 in ff_filter_graph_run_once (graph=0x29d6020) at libavfilter/avfilt
ergraph.c:1449
#11 0x000000000045dd70 in get_frame_internal (ctx=0x2e612e0, frame=0x0, flags=1, samples=0
) at libavfilter/buffersink.c:110
#12 0x000000000045ddcd in av_buffersink_get_frame_flags (ctx=0x2e612e0, frame=0x0, flags=1
) at libavfilter/buffersink.c:121
#13 0x000000000045d916 in avfilter_graph_request_oldest (graph=0x29d6020) at libavfilter/a
vfiltergraph.c:1402
#14 0x000000000042f25a in transcode_from_filter (graph=0x25154c0, best_ist=0x7fffffffdcf8)
 at ffmpeg.c:4455
#15 0x000000000042f511 in transcode_step () at ffmpeg.c:4521
#16 0x000000000042f789 in transcode () at ffmpeg.c:4597
#17 0x000000000042fe75 in main (argc=8, argv=0x7fffffffdf88) at ffmpeg.c:4803

可以看出，17~6都是解码器完成的，filter只完成5~1即可

framesync

ffmpeg里面多路输入都要用framesync来同步
每一个输入在初始化过滤器的时候会有一个权重
权重最大的会触发on_event
on_event是一个callback函数
dualinput不过是利用framesync重写了一遍

filter里面相关函数

filter_frame

static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
{
    OverlayContext *s = inlink->dst->priv;
    //注意这里已经执行过config_input
    av_log(inlink->dst, AV_LOG_DEBUG, "Incoming frame (time:%s) from link #%d\n", av_ts2timestr(inpicref->pts, &inlink->time_base), FF_INLINK_IDX(inlink));
    return ff_dualinput_filter_frame(&s->dinput, inlink, inpicref);
}

ff_dualinput_filter_frame

这个函数在dualinput.c里面

int ff_dualinput_filter_frame(FFDualInputContext *s,
                                   AVFilterLink *inlink, AVFrame *in)
{
    return ff_framesync_filter_frame(&s->fs, inlink, in);
}

直接return ff_framesync_filter_frame

ff_framesync_filter_frame

位于framesync.h里面

int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink,
                              AVFrame *in);

/**
 * Request a frame on the filter output.
 *
 * This function can be the complete implementation of all filter_frame
 * methods of a filter using framesync if it has only one output.
 */

如果多路输入一个输出需要这个函数充当filter_frame()的角色
其实现为

int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink,
                              AVFrame *in)
{
    int ret;

    if ((ret = ff_framesync_process_frame(fs, 1)) < 0)
        return ret;
    if ((ret = ff_framesync_add_frame(fs, FF_INLINK_IDX(inlink), in)) < 0)
        return ret;
    if ((ret = ff_framesync_process_frame(fs, 0)) < 0)
        return ret;
    return 0;
}

这个函数分别执行三个函数，小于0则退出
在vf_overlay.c里面三个都执行了，do_blend是在第三次执行的时候被调用的

ff_framesync_process_frame

ff_framesync_process_frame的定义

int ff_framesync_process_frame(FFFrameSync *fs, unsigned all)
{
    int ret, count = 0;

    av_assert0(fs->on_event);
    while (1) {
        ff_framesync_next(fs);
        if (fs->eof || !fs->frame_ready)
            break;
        if ((ret = fs->on_event(fs)) < 0)
        //在这里调用process_frame，具体在下面说
            return ret;
        ff_framesync_drop(fs);
        count++;
        if (!all)
            break;
    }
    if (!count && fs->eof)
        return AVERROR_EOF;
    return count;
}

注意 FFFrameSync里面有

typedef struct FFFrameSync {
    ......
    int (*on_event)(struct FFFrameSync *fs);

    /**
     * Opaque pointer, not used by the API
     */
    ......
} FFFrameSync;

dualinput.c的代码

static int process_frame(FFFrameSync *fs)
{
    ......
    if (secondpic && !ctx->is_disabled)
        mainpic = s->process(ctx, mainpic, secondpic);
    ret = ff_filter_frame(ctx->outputs[0], mainpic)
}

int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s)
{
    ......
    s->fs.on_event = process_frame;
    ......

ff_dualinput_uninit

对dinput进行初始化，在这里存放不同视频的帧数据

void ff_dualinput_uninit(FFDualInputContext *s)
{
    ff_framesync_uninit(&s->fs);
}

void ff_framesync_uninit(FFFrameSync *fs);

/**
 * Add a frame to an input
 *
 * Typically called from the filter_frame() method.
 *
 * @param fs     frame sync structure
 * @param in     index of the input
 * @param frame  input frame, or NULL for EOF
 */

总结

对于双路输入，自己写过滤器的话，

1. 在filtercontext里面要包含一个FFDualInputContext的结构体
2. uninit进行初始化
3. 定义process回调函数
4. 在filter_frame里面调用ff_dualinput_filter_frame。