FFmpeg源代码简单分析-编码-av_write_frame()

参考链接

• FFmpeg源代码简单分析：av_write_frame()_雷霄骅的博客-CSDN博客_av_write_frame

av_write_frame()

• av_write_frame()用于输出一帧视音频数据，它的声明位于libavformat\avformat.h，如下所示。

/**
 * Write a packet to an output media file.
 *
 * This function passes the packet directly to the muxer, without any buffering
 * or reordering. The caller is responsible for correctly interleaving the
 * packets if the format requires it. Callers that want libavformat to handle
 * the interleaving should call av_interleaved_write_frame() instead of this
 * function.
 *
 * @param s media file handle
 * @param pkt The packet containing the data to be written. Note that unlike
 *            av_interleaved_write_frame(), this function does not take
 *            ownership of the packet passed to it (though some muxers may make
 *            an internal reference to the input packet).
 *            

 *            This parameter can be NULL (at any time, not just at the end), in
 *            order to immediately flush data buffered within the muxer, for
 *            muxers that buffer up data internally before writing it to the
 *            output.
 *            

 *            Packet's @ref AVPacket.stream_index "stream_index" field must be
 *            set to the index of the corresponding stream in @ref
 *            AVFormatContext.streams "s->streams".
 *            

 *            The timestamps (@ref AVPacket.pts "pts", @ref AVPacket.dts "dts")
 *            must be set to correct values in the stream's timebase (unless the
 *            output format is flagged with the AVFMT_NOTIMESTAMPS flag, then
 *            they can be set to AV_NOPTS_VALUE).
 *            The dts for subsequent packets passed to this function must be strictly
 *            increasing when compared in their respective timebases (unless the
 *            output format is flagged with the AVFMT_TS_NONSTRICT, then they
 *            merely have to be nondecreasing).  @ref AVPacket.duration
 *            "duration") should also be set if known.
 * @return < 0 on error, = 0 if OK, 1 if flushed and there is no more data to flush
 *
 * @see av_interleaved_write_frame()
 */
int av_write_frame(AVFormatContext *s, AVPacket *pkt);

• 简单解释一下它的参数的含义：
  • s：用于输出的AVFormatContext。
  • pkt：等待输出的AVPacket。
• 函数正常执行后返回值等于0。 
• av_write_frame()的定义位于libavformat\mux.c，如下所示。

int av_write_frame(AVFormatContext *s, AVPacket *in)
{
    FFFormatContext *const si = ffformatcontext(s);
    AVPacket *pkt = si->parse_pkt;
    int ret;

    if (!in) {
        if (s->oformat->flags & AVFMT_ALLOW_FLUSH) {
            ret = s->oformat->write_packet(s, NULL);
            flush_if_needed(s);
            if (ret >= 0 && s->pb && s->pb->error < 0)
                ret = s->pb->error;
            return ret;
        }
        return 1;
    }

    if (in->flags & AV_PKT_FLAG_UNCODED_FRAME) {
        pkt = in;
    } else {
        /* We don't own in, so we have to make sure not to modify it.
         * (ff_write_chained() relies on this fact.)
         * The following avoids copying in's data unnecessarily.
         * Copying side data is unavoidable as a bitstream filter
         * may change it, e.g. free it on errors. */
        pkt->data = in->data;
        pkt->size = in->size;
        ret = av_packet_copy_props(pkt, in);
        if (ret < 0)
            return ret;
        if (in->buf) {
            pkt->buf = av_buffer_ref(in->buf);
            if (!pkt->buf) {
                ret = AVERROR(ENOMEM);
                goto fail;
            }
        }
    }

    ret = write_packets_common(s, pkt, 0/*non-interleaved*/);

fail:
    // Uncoded frames using the noninterleaved codepath are also freed here
    av_packet_unref(pkt);
    return ret;
}

• 从源代码可以看出，av_write_frame()主要完成了以下几步工作：版本差异
• （1）调用check_packet()做一些简单的检测，新版本目前将check_packet放进write_packets_common函数内
• （2）调用compute_pkt_fields2()设置AVPacket的一些属性值   compute_pkt_fields2函数感觉已被弃用，其功能合并进其余函数
• （3）调用write_packets_common()写入数据

• 如果AVPacket中的flag标记中包含AV_PKT_FLAG_UNCODED_FRAME，就会调用AVOutputFormat的write_uncoded_frame()函数（对应上文代码，pkt=in）；如果不包含那个标记，就会调用write_packet()函数（执行else里面的内容）。 

 av_write_frame

static int write_packets_common(AVFormatContext *s, AVPacket *pkt, int interleaved)
{
    AVStream *st;
    FFStream *sti;
    int ret = check_packet(s, pkt);
    if (ret < 0)
        return ret;
    st = s->streams[pkt->stream_index];
    sti = ffstream(st);

    ret = prepare_input_packet(s, st, pkt);
    if (ret < 0)
        return ret;

    ret = check_bitstream(s, sti, pkt);
    if (ret < 0)
        return ret;

    if (sti->bsfc) {
        return write_packets_from_bsfs(s, st, pkt, interleaved);
    } else {
        return write_packet_common(s, st, pkt, interleaved);
    }
}

check_packet()

• check_packet()定义位于libavformat\mux.c，如下所示。
• 从代码中可以看出，check_packet()的功能比较简单：
• 然后检查一下AVPacket的stream_index（标记了该AVPacket所属的AVStream）设置是否正常，如果为负数或者大于AVStream的个数，则返回错误信息；
• 最后检查AVPacket所属的AVStream是否属于attachment stream，这个地方没见过，目前还没有研究。

static int check_packet(AVFormatContext *s, AVPacket *pkt)
{
    if (pkt->stream_index < 0 || pkt->stream_index >= s->nb_streams) {
        av_log(s, AV_LOG_ERROR, "Invalid packet stream index: %d\n",
               pkt->stream_index);
        return AVERROR(EINVAL);
    }

    if (s->streams[pkt->stream_index]->codecpar->codec_type == AVMEDIA_TYPE_ATTACHMENT) {
        av_log(s, AV_LOG_ERROR, "Received a packet for an attachment stream.\n");
        return AVERROR(EINVAL);
    }

    return 0;
}

AVOutputFormat->write_packet()

• write_packet()函数的定义位于libavformat\mux.c，如下所示。

    /**
     * Write a packet. If AVFMT_ALLOW_FLUSH is set in flags,
     * pkt can be NULL in order to flush data buffered in the muxer.
     * When flushing, return 0 if there still is more data to flush,
     * or 1 if everything was flushed and there is no more buffered
     * data.
     */
    int (*write_packet)(struct AVFormatContext *, AVPacket *pkt);

static int write_packet(AVFormatContext *s1, AVPacket *pkt)
{
    const V4L2Context *s = s1->priv_data;
    if (write(s->fd, pkt->data, pkt->size) == -1)
        return AVERROR(errno);
    return 0;
}

• write_packet()实际上是一个函数指针，指向特定的AVOutputFormat中的实现函数。
• 例如，我们看一下FLV对应的AVOutputFormat，位于libavformat\flvenc.c，如下所示。

const AVOutputFormat ff_flv_muxer = {
    .name           = "flv",
    .long_name      = NULL_IF_CONFIG_SMALL("FLV (Flash Video)"),
    .mime_type      = "video/x-flv",
    .extensions     = "flv",
    .priv_data_size = sizeof(FLVContext),
    .audio_codec    = CONFIG_LIBMP3LAME ? AV_CODEC_ID_MP3 : AV_CODEC_ID_ADPCM_SWF,
    .video_codec    = AV_CODEC_ID_FLV1,
    .init           = flv_init,
    .write_header   = flv_write_header,
    .write_packet   = flv_write_packet,
    .write_trailer  = flv_write_trailer,
    .check_bitstream= flv_check_bitstream,
    .codec_tag      = (const AVCodecTag* const []) {
                          flv_video_codec_ids, flv_audio_codec_ids, 0
                      },
    .flags          = AVFMT_GLOBALHEADER | AVFMT_VARIABLE_FPS |
                      AVFMT_TS_NONSTRICT,
    .priv_class     = &flv_muxer_class,
};

• 从ff_flv_muxer的定义可以看出，write_packet()指向的是flv_write_packet()函数。
• 在看flv_write_packet()函数的定义之前，我们先回顾一下FLV封装格式的结构。

FLV封装格式

• FLV封装格式如下图所示。
• PS：原图是网上找的，感觉画的很清晰，比官方的Video File Format Specification更加通俗易懂。但是图中有一个错误，就是TagHeader中的StreamID字段的长度写错了（查看了一下官方标准，应该是3字节，现在已经改过来了）。

• 从FLV的封装格式结构可以看出，它的文件数据是一个一个的Tag连接起来的，中间间隔包含着Previous Tag Size。
• 因此，flv_write_packet()函数的任务就是写入一个Tag和Previous Tag Size。
• 下面简单记录一下Tag Data的格式。
• Tag Data根据Tag的Type不同而不同：可以分为音频Tag Data，视频Tag Data以及Script Tag Data。
• 下面简述一下音频Tag Data和视频Tag Data。

Audio Tag Data

• Audio Tag在官方标准中定义如下

• Audio Tag开始的第1个字节包含了音频数据的参数信息，从第2个字节开始为音频流数据。
• 第1个字节的前4位的数值表示了音频数据格式：
  • 0 = Linear PCM, platform endian
  • 1 = ADPCM 2 = MP3
  • 3 = Linear PCM, little endian
  • 4 = Nellymoser 16-kHz mono
  • 5 = Nellymoser 8-kHz mono
  • 6 = Nellymoser
  • 7 = G.711 A-law logarithmic PCM
  • 8 = G.711 mu-law logarithmic PCM
  • 9 = reserved
  • 10 = AAC
  • 14 = MP3 8-Khz
  • 15 = Device-specific sound
• 第1个字节的第5-6位的数值表示采样率：0 = 5.5kHz，1 = 11KHz，2 = 22 kHz，3 = 44 kHz。
• 第1个字节的第7位表示采样精度：0 = 8bits，1 = 16bits。
• 第1个字节的第8位表示音频类型：0 = sndMono，1 = sndStereo。
• 其中，当音频编码为AAC的时候，第一个字节后面存储的是AACAUDIODATA，格式如下所示。 

 Video Tag Data

• Video Tag在官方标准中的定义如下

•  Video Tag也用开始的第1个字节包含视频数据的参数信息，从第2个字节为视频流数据。
• 第1个字节的前4位的数值表示帧类型（FrameType）：
  • 1: keyframe (for AVC, a seekableframe)（关键帧）
  • 2: inter frame (for AVC, a nonseekableframe)
  • 3: disposable inter frame (H.263only)
  • 4: generated keyframe (reservedfor server use only)
  • 5: video info/command frame
• 第1个字节的后4位的数值表示视频编码ID（CodecID）：
  • 1: JPEG (currently unused)
  • 2: Sorenson H.263
  • 3: Screen video
  • 4: On2 VP6
  • 5: On2 VP6 with alpha channel
  • 6: Screen video version 2
  • 7: AVC
• 其中，当音频编码为AVC（H.264）的时候，第一个字节后面存储的是AVCVIDEOPACKET，格式如下所示

 flv_write_packet()

• 下面我们看一下FLV格式中write_packet()对应的实现函数flv_write_packet()的定义，位于libavformat\flvenc.c，如下所示。

static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
{
    AVIOContext *pb      = s->pb;
    AVCodecParameters *par = s->streams[pkt->stream_index]->codecpar;
    FLVContext *flv      = s->priv_data;
    FLVStreamContext *sc = s->streams[pkt->stream_index]->priv_data;
    unsigned ts;
    int size = pkt->size;
    uint8_t *data = NULL;
    int flags = -1, flags_size, ret = 0;
    int64_t cur_offset = avio_tell(pb);

    if (par->codec_type == AVMEDIA_TYPE_AUDIO && !pkt->size) {
        av_log(s, AV_LOG_WARNING, "Empty audio Packet\n");
        return AVERROR(EINVAL);
    }

    if (par->codec_id == AV_CODEC_ID_VP6F || par->codec_id == AV_CODEC_ID_VP6A ||
        par->codec_id == AV_CODEC_ID_VP6  || par->codec_id == AV_CODEC_ID_AAC)
        flags_size = 2;
    else if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_MPEG4)
        flags_size = 5;
    else
        flags_size = 1;

    if (par->codec_id == AV_CODEC_ID_AAC || par->codec_id == AV_CODEC_ID_H264
            || par->codec_id == AV_CODEC_ID_MPEG4) {
        size_t side_size;
        uint8_t *side = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
        if (side && side_size > 0 && (side_size != par->extradata_size || memcmp(side, par->extradata, side_size))) {
            ret = ff_alloc_extradata(par, side_size);
            if (ret < 0)
                return ret;
            memcpy(par->extradata, side, side_size);
            flv_write_codec_header(s, par, pkt->dts);
        }
    }

    if (flv->delay == AV_NOPTS_VALUE)
        flv->delay = -pkt->dts;

    if (pkt->dts < -flv->delay) {
        av_log(s, AV_LOG_WARNING,
               "Packets are not in the proper order with respect to DTS\n");
        return AVERROR(EINVAL);
    }
    if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_MPEG4) {
        if (pkt->pts == AV_NOPTS_VALUE) {
            av_log(s, AV_LOG_ERROR, "Packet is missing PTS\n");
            return AVERROR(EINVAL);
        }
    }

    ts = pkt->dts;

    if (s->event_flags & AVSTREAM_EVENT_FLAG_METADATA_UPDATED) {
        write_metadata(s, ts);
        s->event_flags &= ~AVSTREAM_EVENT_FLAG_METADATA_UPDATED;
    }

    avio_write_marker(pb, av_rescale(ts, AV_TIME_BASE, 1000),
                      pkt->flags & AV_PKT_FLAG_KEY && (flv->video_par ? par->codec_type == AVMEDIA_TYPE_VIDEO : 1) ? AVIO_DATA_MARKER_SYNC_POINT : AVIO_DATA_MARKER_BOUNDARY_POINT);

    switch (par->codec_type) {
    case AVMEDIA_TYPE_VIDEO:
        avio_w8(pb, FLV_TAG_TYPE_VIDEO);

        flags = ff_codec_get_tag(flv_video_codec_ids, par->codec_id);

        flags |= pkt->flags & AV_PKT_FLAG_KEY ? FLV_FRAME_KEY : FLV_FRAME_INTER;
        break;
    case AVMEDIA_TYPE_AUDIO:
        flags = get_audio_flags(s, par);

        av_assert0(size);

        avio_w8(pb, FLV_TAG_TYPE_AUDIO);
        break;
    case AVMEDIA_TYPE_SUBTITLE:
    case AVMEDIA_TYPE_DATA:
        avio_w8(pb, FLV_TAG_TYPE_META);
        break;
    default:
        return AVERROR(EINVAL);
    }

    if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_MPEG4) {
        /* check if extradata looks like mp4 formatted */
        if (par->extradata_size > 0 && *(uint8_t*)par->extradata != 1)
            if ((ret = ff_avc_parse_nal_units_buf(pkt->data, &data, &size)) < 0)
                return ret;
    } else if (par->codec_id == AV_CODEC_ID_AAC && pkt->size > 2 &&
               (AV_RB16(pkt->data) & 0xfff0) == 0xfff0) {
        if (!s->streams[pkt->stream_index]->nb_frames) {
            av_log(s, AV_LOG_ERROR, "Malformed AAC bitstream detected: "
                   "use the audio bitstream filter 'aac_adtstoasc' to fix it "
                   "('-bsf:a aac_adtstoasc' option with ffmpeg)\n");
            return AVERROR_INVALIDDATA;
        }
        av_log(s, AV_LOG_WARNING, "aac bitstream error\n");
    }

    /* check Speex packet duration */
    if (par->codec_id == AV_CODEC_ID_SPEEX && ts - sc->last_ts > 160)
        av_log(s, AV_LOG_WARNING, "Warning: Speex stream has more than "
                                  "8 frames per packet. Adobe Flash "
                                  "Player cannot handle this!\n");

    if (sc->last_ts < ts)
        sc->last_ts = ts;

    if (size + flags_size >= 1<<24) {
        av_log(s, AV_LOG_ERROR, "Too large packet with size %u >= %u\n",
               size + flags_size, 1<<24);
        ret = AVERROR(EINVAL);
        goto fail;
    }

    avio_wb24(pb, size + flags_size);
    put_timestamp(pb, ts);
    avio_wb24(pb, flv->reserved);

    if (par->codec_type == AVMEDIA_TYPE_DATA ||
        par->codec_type == AVMEDIA_TYPE_SUBTITLE ) {
        int data_size;
        int64_t metadata_size_pos = avio_tell(pb);
        if (par->codec_id == AV_CODEC_ID_TEXT) {
            // legacy FFmpeg magic?
            avio_w8(pb, AMF_DATA_TYPE_STRING);
            put_amf_string(pb, "onTextData");
            avio_w8(pb, AMF_DATA_TYPE_MIXEDARRAY);
            avio_wb32(pb, 2);
            put_amf_string(pb, "type");
            avio_w8(pb, AMF_DATA_TYPE_STRING);
            put_amf_string(pb, "Text");
            put_amf_string(pb, "text");
            avio_w8(pb, AMF_DATA_TYPE_STRING);
            put_amf_string(pb, pkt->data);
            put_amf_string(pb, "");
            avio_w8(pb, AMF_END_OF_OBJECT);
        } else {
            // just pass the metadata through
            avio_write(pb, data ? data : pkt->data, size);
        }
        /* write total size of tag */
        data_size = avio_tell(pb) - metadata_size_pos;
        avio_seek(pb, metadata_size_pos - 10, SEEK_SET);
        avio_wb24(pb, data_size);
        avio_seek(pb, data_size + 10 - 3, SEEK_CUR);
        avio_wb32(pb, data_size + 11);
    } else {
        av_assert1(flags>=0);
        avio_w8(pb,flags);
        if (par->codec_id == AV_CODEC_ID_VP6)
            avio_w8(pb,0);
        if (par->codec_id == AV_CODEC_ID_VP6F || par->codec_id == AV_CODEC_ID_VP6A) {
            if (par->extradata_size)
                avio_w8(pb, par->extradata[0]);
            else
                avio_w8(pb, ((FFALIGN(par->width,  16) - par->width) << 4) |
                             (FFALIGN(par->height, 16) - par->height));
        } else if (par->codec_id == AV_CODEC_ID_AAC)
            avio_w8(pb, 1); // AAC raw
        else if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_MPEG4) {
            avio_w8(pb, 1); // AVC NALU
            avio_wb24(pb, pkt->pts - pkt->dts);
        }

        avio_write(pb, data ? data : pkt->data, size);

        avio_wb32(pb, size + flags_size + 11); // previous tag size
        flv->duration = FFMAX(flv->duration,
                              pkt->pts + flv->delay + pkt->duration);
    }

    if (flv->flags & FLV_ADD_KEYFRAME_INDEX) {
        switch (par->codec_type) {
            case AVMEDIA_TYPE_VIDEO:
                flv->videosize += (avio_tell(pb) - cur_offset);
                flv->lasttimestamp = flv->acurframeindex / flv->framerate;
                flv->acurframeindex++;
                if (pkt->flags & AV_PKT_FLAG_KEY) {
                    double ts = flv->lasttimestamp;
                    int64_t pos = cur_offset;

                    flv->lastkeyframetimestamp = ts;
                    flv->lastkeyframelocation = pos;
                    ret = flv_append_keyframe_info(s, flv, ts, pos);
                    if (ret < 0)
                        goto fail;
                }
                break;

            case AVMEDIA_TYPE_AUDIO:
                flv->audiosize += (avio_tell(pb) - cur_offset);
                break;

            default:
                av_log(s, AV_LOG_WARNING, "par->codec_type is type = [%d]\n", par->codec_type);
                break;
        }
    }
fail:
    av_free(data);

    return ret;
}

• 我们通过源代码简单梳理一下flv_write_packet()在写入H.264/AAC时候的流程：
• （1）写入Tag Header的Type，如果是视频，代码如下：avio_w8(pb, FLV_TAG_TYPE_VIDEO);   如果是音频，代码如下： avio_w8(pb, FLV_TAG_TYPE_AUDIO);
• （2）写入Tag Header的Datasize，Timestamp和StreamID（至此完成Tag Header）：

    //Tag Header - Datasize
    avio_wb24(pb, size + flags_size);
         //Tag Header - Timestamp   无定义
         avio_wb24(pb, ts & 0xFFFFFF);
         avio_w8(pb, (ts >> 24) & 0x7F); // timestamps are 32 bits _signed_
    //StreamID
    avio_wb24(pb, flv->reserved);

• （3）写入Tag Data的第一字节（其中flag已经在前面的代码中设置完毕）：
• //First Byte of Tag Data
• avio_w8(pb,flags);
• （4）如果编码格式VP6作相应的处理（不研究）；编码格式为AAC，写入AACAUDIODATA；编码格式为H.264，写入AVCVIDEOPACKET：

        if (enc->codec_id == AV_CODEC_ID_VP6F || enc->codec_id == AV_CODEC_ID_VP6A) {
            if (enc->extradata_size)
                avio_w8(pb, enc->extradata[0]);
            else
                avio_w8(pb, ((FFALIGN(enc->width,  16) - enc->width) << 4) |
                             (FFALIGN(enc->height, 16) - enc->height));
        } else if (enc->codec_id == AV_CODEC_ID_AAC)
            avio_w8(pb, 1); // AAC raw
        else if (enc->codec_id == AV_CODEC_ID_H264 || enc->codec_id == AV_CODEC_ID_MPEG4) {
            //AVCVIDEOPACKET-AVCPacketType
        	avio_w8(pb, 1); // AVC NALU
        	//AVCVIDEOPACKET-CompositionTime
            avio_wb24(pb, pkt->pts - pkt->dts);
        }

• （5）写入数据：avio_write(pb, data ? data : pkt->data, size);  //Data
• （6）  写入previous tag size：avio_wb32(pb, size + flags_size + 11); // previous tag size
• 至此，flv_write_packet()就完成了一个Tag的写入。