FFmpeg源码分析：avcodec_send_frame()和avcodec_receive_packet()音视频编码

FFmpeg在libavcodec模块，旧版本提供avcodec_encode_video2()作为视频编码函数，avcodec_encode_audio2()作为音频编码函数。在FFmpeg 3.1版本新增avcodec_send_frame()与avcodec_receive_packet()作为音视频编码函数。后来，在3.4版本把avcodec_encode_video2()和avcodec_encode_audio2()标记为过时API。

在上一篇文章介绍到音视频解码的分析：avcodec_send_packet和avcodec_receive_frame。

目录

一、avcodec_send_frame发送AVFrame

1、avcodec_send_frame

二、avcodec_receive_packet接收AVPacket

1、avcodec_receive_packet

2、encode_receive_packet_internal

3、encode_simple_receive_packet

4、encode_simple_internal

5、X264_frame

三、avcodec_encode_video2视频编码

四、avcodec_encode_audio2音频编码

---

avcodec_send_frame和avcodec_receive_packet函数定义位于libavcodec/avcodec.h：

/**
 * Supply a raw video or audio frame to the encoder. Use avcodec_receive_packet()
 * to retrieve buffered output packets.
 *
 * @param avctx     codec context
 * @param[in] frame AVFrame containing the raw audio or video frame to be encoded.
 *
 *                  For audio:
 *                  If AV_CODEC_CAP_VARIABLE_FRAME_SIZE is set, then each frame
 *                  can have any number of samples.
 * @return 0 on success, otherwise negative error code:
 *      AVERROR(EAGAIN):   input is not accepted in the current state, try again.
 *      AVERROR_EOF:       the encoder has been flushed, and no new frames.
 *      AVERROR(EINVAL):   codec not opened, refcounted_frames not set, it is a
 *                         decoder, or requires flush
 *      AVERROR(ENOMEM):   failed to add packet to internal queue, or similar
 *      other errors: legitimate encoding errors
 */
int avcodec_send_frame(AVCodecContext *avctx, const AVFrame *frame);

/**
 * Read encoded data from the encoder.
 *
 * @param avctx codec context
 * @param avpkt This will be set to a reference-counted packet
 *
 * @return 0 on success, otherwise negative error code:
 *      AVERROR(EAGAIN):   output is not available in the current state, try again.
 *      AVERROR_EOF:       the encoder has been fully flushed, and there will be
 *                         no more output packets
 *      AVERROR(EINVAL):   codec not opened, or it is a decoder
 *      other errors: legitimate encoding errors
 */
int avcodec_receive_packet(AVCodecContext *avctx, AVPacket *avpkt);

由描述可知， avcodec_send_frame()负责给编码器提供未压缩的原始数据，avcodec_receive_packet()则从编码器取出编码后的数据。如果返回0，代表成功；返回AGAIN，代表当前状态没有可输出数据；返回EOF，代表已经到达输入流结尾；返回INVAL，代表编码器没有打开或者打开的是解码器。

音视频编码的函数执行流程图如下：

一、avcodec_send_frame发送AVFrame

1、avcodec_send_frame

avcodec_send_frame()函数位于libavcodec/encode.c，首先判断编码器有没打开、是否为编码器，然后调用internal函数执行具体操作：

int avcodec_send_frame(AVCodecContext *avctx, const AVFrame *frame)
{
    AVCodecInternal *avci = avctx->internal;
    int ret;
    // 判断编码器有没打开，是否为编码器
    if (!avcodec_is_open(avctx) || !av_codec_is_encoder(avctx->codec))
        return AVERROR(EINVAL);
    if (avci->draining)
        return AVERROR_EOF;
    if (avci->buffer_frame->data[0])
        return AVERROR(EAGAIN);
    if (!frame) {
        avci->draining = 1;
    } else {
        ret = encode_send_frame_internal(avctx, frame);
        if (ret < 0)
            return ret;
    }

    if (!avci->buffer_pkt->data && !avci->buffer_pkt->side_data) {
        ret = encode_receive_packet_internal(avctx, avci->buffer_pkt);
        if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
            return ret;
    }

    return 0;
}

 encode_send_frame_internal()函数没做多少事情，只是解析音频metadata、检查frame是否有效，真正的赋值操作不在这里。代码如下：

static int encode_send_frame_internal(AVCodecContext *avctx, const AVFrame *src)
{
    AVCodecInternal *avci = avctx->internal;
    AVFrame *dst = avci->buffer_frame;
    int ret;

    if (avctx->codec->type == AVMEDIA_TYPE_AUDIO) {
        // 解析音频metadata
        AVFrameSideData *sd = av_frame_get_side_data(src, AV_FRAME_DATA_AUDIO_SERVICE_TYPE);
        if (sd && sd->size >= sizeof(enum AVAudioServiceType))
            avctx->audio_service_type = *(enum AVAudioServiceType*)sd->data;
        // 检查frame大小是否有效
        if (avctx->codec->capabilities & AV_CODEC_CAP_SMALL_LAST_FRAME) {
            if (src->nb_samples > avctx->frame_size) {
                return AVERROR(EINVAL);
            }
        } else if (!(avctx->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)) {
            if (avctx->internal->last_audio_frame) {
                return AVERROR(EINVAL);
            }
            if (src->nb_samples < avctx->frame_size) {
                ret = pad_last_frame(avctx, dst, src);
                if (ret < 0)
                    return ret;
                avctx->internal->last_audio_frame = 1;
            } else if (src->nb_samples > avctx->frame_size) {
                return AVERROR(EINVAL);
            }
        }
    }
    if (!dst->data[0]) {
        ret = av_frame_ref(dst, src);
        if (ret < 0)
             return ret;
    }

    return 0;
}

二、avcodec_receive_packet接收AVPacket

1、avcodec_receive_packet

avcodec_receive_packet()函数也是首先编码器是否打开、是否为编码器，然后调用encode_receive_packet_internal()函数去执行具体操作：

int avcodec_receive_packet(AVCodecContext *avctx, AVPacket *avpkt)
{
    AVCodecInternal *avci = avctx->internal;
    int ret;
    av_packet_unref(avpkt);
    // 判断编码器是否打开，是否为编码器
    if (!avcodec_is_open(avctx) || !av_codec_is_encoder(avctx->codec))
        return AVERROR(EINVAL);

    if (avci->buffer_pkt->data || avci->buffer_pkt->side_data) {
        av_packet_move_ref(avpkt, avci->buffer_pkt);
    } else {
        ret = encode_receive_packet_internal(avctx, avpkt);
        if (ret < 0)
            return ret;
    }

    return 0;
}

2、encode_receive_packet_internal

encode_receive_packet_internal()函数首先检测视频宽高、像素格式，然后判断使用receive_packet还是encode_simple_receive_packet执行编码操作：

static int encode_receive_packet_internal(AVCodecContext *avctx, AVPacket *avpkt)
{
    AVCodecInternal *avci = avctx->internal;
    int ret;

    if (avci->draining_done)
        return AVERROR_EOF;
    // 检查视频宽、高、像素格式
    if (avctx->codec->type == AVMEDIA_TYPE_VIDEO) {
        if (av_image_check_size2(avctx->width, avctx->height, 
			avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx))
            return AVERROR(EINVAL);
    }
    // 判断使用receive_packet还是encode_simple_receive_packet编码
    if (avctx->codec->receive_packet) {
        ret = avctx->codec->receive_packet(avctx, avpkt);
        if (ret < 0)
            av_packet_unref(avpkt);
        else
            av_assert0(!avpkt->data || avpkt->buf);
    } else
        ret = encode_simple_receive_packet(avctx, avpkt);

    if (ret == AVERROR_EOF)
        avci->draining_done = 1;

    return ret;
}

3、encode_simple_receive_packet

encode_simple_receive_packet()函数比较简单，主要是调用encode_simple_internal()函数：

static int encode_simple_receive_packet(AVCodecContext *avctx, AVPacket *avpkt)
{
    int ret;

    while (!avpkt->data && !avpkt->side_data) {
        ret = encode_simple_internal(avctx, avpkt);
        if (ret < 0)
            return ret;
    }

    return 0;
}

4、encode_simple_internal

encode_simple_internal()函数首先判断frame，如果frame为空则调用ff_encode_get_frame()取出一帧未压缩的数据，然后判断使用ff_thread_video_encode_frame还是avctx->codec->encode2执行真正的编码操作：

static int encode_simple_internal(AVCodecContext *avctx, AVPacket *avpkt)
{
    AVCodecInternal   *avci = avctx->internal;
    EncodeSimpleContext *es = &avci->es;
    AVFrame          *frame = es->in_frame;
    int got_packet;
    int ret;

    if (avci->draining_done)
        return AVERROR_EOF;
    // 如果frame为空，调用ff_encode_get_frame取一帧数据
    if (!frame->buf[0] && !avci->draining) {
        av_frame_unref(frame);
        ret = ff_encode_get_frame(avctx, frame);
        if (ret < 0 && ret != AVERROR_EOF)
            return ret;
    }
    if (!frame->buf[0]) {
        if (!(avctx->codec->capabilities & AV_CODEC_CAP_DELAY ||
              (avci->frame_thread_encoder && avctx->active_thread_type 
			  & FF_THREAD_FRAME)))
            return AVERROR_EOF;
        frame = NULL;
    }
    got_packet = 0;
    // 判断使用ff_thread_video_encode_frame还是avctx->codec->encode2进行编码
    if (CONFIG_FRAME_THREAD_ENCODER &&
        avci->frame_thread_encoder && (avctx->active_thread_type & FF_THREAD_FRAME))
        ret = ff_thread_video_encode_frame(avctx, avpkt, frame, &got_packet);
    else {
        ret = avctx->codec->encode2(avctx, avpkt, frame, &got_packet);
        if (avctx->codec->type == AVMEDIA_TYPE_VIDEO && !ret && got_packet &&
            !(avctx->codec->capabilities & AV_CODEC_CAP_DELAY))
            avpkt->pts = avpkt->dts = frame->pts;
    }

    if (!ret && got_packet) {
        if (avpkt->data) {
            ret = av_packet_make_refcounted(avpkt);
            if (ret < 0)
                goto end;
        }
        if (frame && !(avctx->codec->capabilities & AV_CODEC_CAP_DELAY)) {
            if (avctx->codec->type == AVMEDIA_TYPE_AUDIO) {
                if (avpkt->pts == AV_NOPTS_VALUE)
                    avpkt->pts = frame->pts;
                if (!avpkt->duration)
                    avpkt->duration = ff_samples_to_time_base(
				        avctx, frame->nb_samples);
            }
        }
        // 如果是音频，flags都设为关键帧
        if (avctx->codec->type == AVMEDIA_TYPE_AUDIO) {
            avpkt->flags |= AV_PKT_FLAG_KEY;
            avpkt->dts = avpkt->pts;
        }
    }

    if (avci->draining && !got_packet)
        avci->draining_done = 1;

end:
    if (ret < 0 || !got_packet)
        av_packet_unref(avpkt);
    if (frame) {
        if (!ret)
            avctx->frame_number++;
        av_frame_unref(frame);
    }

    return ret;
}

 在上面有提到，avcodec_send_frame()函数没有进行frame的赋值操作，真正执行赋值操作的是ff_encode_get_frame()函数，具体如下：

int ff_encode_get_frame(AVCodecContext *avctx, AVFrame *frame)
{
    AVCodecInternal *avci = avctx->internal;

    if (avci->draining)
        return AVERROR_EOF;
    if (!avci->buffer_frame->buf[0])
        return AVERROR(EAGAIN);
    // avci->buffer_frame赋值给frame
    av_frame_move_ref(frame, avci->buffer_frame);

    return 0;
}

5、X264_frame

以libx264编码器为例，位于libavcodec/libx264.c，对应的AVCodec如下：

AVCodec ff_libx264_encoder = {
    .name             = "libx264",
    .long_name        = NULL_IF_CONFIG_SMALL("libx264 H.264/AVC /MPEG-4 part10"),
    .type             = AVMEDIA_TYPE_VIDEO,
    .id               = AV_CODEC_ID_H264,
    .priv_data_size   = sizeof(X264Context),
    .init             = X264_init,
    .encode2          = X264_frame,
    .close            = X264_close,
    .capabilities     = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_OTHER_THREADS |
                        AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE,
    .caps_internal    = FF_CODEC_CAP_AUTO_THREADS,
    .priv_class       = &x264_class,
    .defaults         = x264_defaults,
    .pix_fmts         = pix_fmts_all,
    .wrapper_name     = "libx264",
};

此时，encode2函数指针指向X264_frame，代码如下：

static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame,
                      int *got_packet)
{
    X264Context *x4 = ctx->priv_data;
    x264_nal_t *nal;
    int nnal, i, ret;
    x264_picture_t pic_out = {0};
    int pict_type;
    int bit_depth;
    int64_t wallclock = 0;
    X264Opaque *out_opaque;
    AVFrameSideData *sd;
    // 初始化x264编码器相关参数
    x264_picture_init( &x4->pic );
    x4->pic.img.i_csp = x4->params.i_csp;
#if X264_BUILD >= 153
    bit_depth = x4->params.i_bitdepth;
#else
    bit_depth = x264_bit_depth;
#endif
    if (bit_depth > 8)
        x4->pic.img.i_csp |= X264_CSP_HIGH_DEPTH;
    x4->pic.img.i_plane = avfmt2_num_planes(ctx->pix_fmt);

    if (frame) {
        for (i = 0; i < x4->pic.img.i_plane; i++) {
            x4->pic.img.plane[i]    = frame->data[i];
            x4->pic.img.i_stride[i] = frame->linesize[i];
        }

        x4->pic.i_pts  = frame->pts;
        x4->reordered_opaque[x4->next_reordered_opaque].reordered_opaque = frame->reordered_opaque;
        x4->reordered_opaque[x4->next_reordered_opaque].wallclock = wallclock;
        if (ctx->export_side_data & AV_CODEC_EXPORT_DATA_PRFT)
            x4->reordered_opaque[x4->next_reordered_opaque].wallclock = av_gettime();
        x4->pic.opaque = &x4->reordered_opaque[x4->next_reordered_opaque];
        x4->next_reordered_opaque++;
        x4->next_reordered_opaque %= x4->nb_reordered_opaque;
        // 给x264的pic.i_type赋值，包括IDR、I、P、B帧类型
        switch (frame->pict_type) {
        case AV_PICTURE_TYPE_I:
            x4->pic.i_type = x4->forced_idr > 0 ? X264_TYPE_IDR
                                                : X264_TYPE_KEYFRAME;
            break;
        case AV_PICTURE_TYPE_P:
            x4->pic.i_type = X264_TYPE_P;
            break;
        case AV_PICTURE_TYPE_B:
            x4->pic.i_type = X264_TYPE_B;
            break;
        default:
            x4->pic.i_type = X264_TYPE_AUTO;
            break;
        }
		// 重新配置编码器
        reconfig_encoder(ctx, frame);

        ......
    }

    do {
		// 调用x264_encoder_encode()执行编码
        if (x264_encoder_encode(x4->enc, &nal, &nnal, frame? &x4->pic: NULL, &pic_out) < 0)
            return AVERROR_EXTERNAL;
        // 对nal单元进行编码
        ret = encode_nals(ctx, pkt, nal, nnal);
        if (ret < 0)
            return ret;
    } while (!ret && !frame && x264_encoder_delayed_frames(x4->enc));

    if (!ret)
        return 0;

    pkt->pts = pic_out.i_pts;
    pkt->dts = pic_out.i_dts;
    out_opaque = pic_out.opaque;
    if (out_opaque >= x4->reordered_opaque &&
        out_opaque < &x4->reordered_opaque[x4->nb_reordered_opaque]) {
        ctx->reordered_opaque = out_opaque->reordered_opaque;
        wallclock = out_opaque->wallclock;
    } else {
        ctx->reordered_opaque = 0;
    }
    // 给pict_type赋值，包括I、P、B帧类型
    switch (pic_out.i_type) {
    case X264_TYPE_IDR:
    case X264_TYPE_I:
        pict_type = AV_PICTURE_TYPE_I;
        break;
    case X264_TYPE_P:
        pict_type = AV_PICTURE_TYPE_P;
        break;
    case X264_TYPE_B:
    case X264_TYPE_BREF:
        pict_type = AV_PICTURE_TYPE_B;
        break;
    default:
        return AVERROR_EXTERNAL;
    }
    pkt->flags |= AV_PKT_FLAG_KEY*pic_out.b_keyframe;
    if (ret) {
        ff_side_data_set_encoder_stats(pkt, 
			(pic_out.i_qpplus1 - 1) * FF_QP2LAMBDA, NULL, 0, pict_type);
        if (wallclock)
            ff_side_data_set_prft(pkt, wallclock);
    }

    *got_packet = ret;
    return 0;
}

由此可见，X264_frame()函数主要有6个步骤：

• 初始化x264编码器相关参数；
• 给x264的pic.i_type赋值，包括IDR、I、P、B帧类型；
• 调用reconfig_encoder()重新配置编码器；
• 调用x264_encoder_encode()执行编码；
• 对nal单元进行编码；
• 给输出的pict_type赋值，包括I、P、B帧类型；

三、avcodec_encode_video2视频编码

由于avcodec_encode_video2()函数已经过时，所以内部提供compat_encode()来兼容旧版本，具体代码如下：

int avcodec_encode_video2(AVCodecContext *avctx,
                                              AVPacket *avpkt,
                                              const AVFrame *frame,
                                              int *got_packet_ptr)
{
    int ret = compat_encode(avctx, avpkt, got_packet_ptr, frame);

    if (ret < 0)
        av_packet_unref(avpkt);

    return ret;
}

我们来看看compat_encode()函数源码，其实内部也是调用avcodec_send_frame()和avcodec_receive_packet()进行编码，具体如下： 

static int compat_encode(AVCodecContext *avctx, AVPacket *avpkt,
                         int *got_packet, const AVFrame *frame)
{
    AVCodecInternal *avci = avctx->internal;
    AVPacket user_pkt;
    int ret;
    *got_packet = 0;
    // 检测视频的pixel_format、width、height
    if (frame && avctx->codec->type == AVMEDIA_TYPE_VIDEO) {
        if (frame->format == AV_PIX_FMT_NONE)
            av_log(avctx, AV_LOG_WARNING, "format is not set\n");
        if (frame->width == 0 || frame->height == 0)
            av_log(avctx, AV_LOG_WARNING, "width or height is not set\n");
    }
    // 调用avcodec_send_frame()发送frame
    ret = avcodec_send_frame(avctx, frame);
    if (ret == AVERROR_EOF)
        ret = 0;
    else if (ret == AVERROR(EAGAIN)) {
        return AVERROR_BUG;
    } else if (ret < 0)
        return ret;

    av_packet_move_ref(&user_pkt, avpkt);
    while (ret >= 0) {
		// 调用avcodec_receive_packet()接收packet
        ret = avcodec_receive_packet(avctx, avpkt);
        if (ret < 0) {
            if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
                ret = 0;
            goto finish;
        }

        if (avpkt != avci->compat_encode_packet) {
            if (avpkt->data && user_pkt.data) {
                if (user_pkt.size >= avpkt->size) {
                    memcpy(user_pkt.data, avpkt->data, avpkt->size);
                    av_buffer_unref(&avpkt->buf);
                    avpkt->buf  = user_pkt.buf;
                    avpkt->data = user_pkt.data;
                } else {
                    av_packet_unref(avpkt);
                    ret = AVERROR(EINVAL);
                    goto finish;
                }
            }

            *got_packet = 1;
            avpkt = avci->compat_encode_packet;
        } else {
            if (!avci->compat_decode_warned) {
                avci->compat_decode_warned = 1;
                av_packet_unref(avpkt);
            }
        }
        if (avci->draining)
            break;
    }

finish:
    if (ret < 0)
        av_packet_unref(&user_pkt);

    return ret;
}

四、avcodec_encode_audio2音频编码

和avcodec_encode_video2()函数一样，avcodec_encode_audio2()函数已经过时，内部也是提供compat_encode()来兼容旧版本，具体代码如下：

int avcodec_encode_audio2(AVCodecContext *avctx,
                                              AVPacket *avpkt,
                                              const AVFrame *frame,
                                              int *got_packet_ptr)
{
    int ret = compat_encode(avctx, avpkt, got_packet_ptr, frame);

    if (ret < 0)
        av_packet_unref(avpkt);

    return ret;
}

至此，avcodec_send_frame()和avcodec_receive_packet()组成的编码函数已经分析完毕。

学习FFmpeg与代码实践，可参考：FFmpegAndroid