FFmpeg 开发系列连载：
- FFmpeg 开发(01)：FFmpeg 编译和集成
- FFmpeg 开发(02)：FFmpeg + ANativeWindow 实现视频解码播放
- FFmpeg 开发(03)：FFmpeg + OpenSLES 实现音频解码播放
- FFmpeg 开发(04)：FFmpeg + OpenGLES 实现音频可视化播放
- FFmpeg 开发(05)：FFmpeg + OpenGLES 实现视频解码播放和视频滤镜
- FFmpeg 开发(07)：FFmpeg + OpenGLES 实现 3D 全景播放器
- FFmpeg 开发(08)：FFmpeg 播放器视频渲染优化
- FFmpeg 开发(09)：FFmpeg、x264以及fdk-aac 编译整合
- FFmpeg 开发(10)：FFmpeg 视频录制 - 视频添加滤镜和编码
- FFmpeg 开发(11)：FFmpeg + Android AudioRecorder 音频录制编码
- FFmpeg 开发(12)：Android FFmpeg 实现带滤镜的小视频录制功能
- FFmpeg 开发(13)：Android FFmpeg 流媒体边播放边录制功能
前面 FFmpeg 系列的文章中，已经实现了音视频的播放、录制已经添加滤镜等功能，本文将用 FFmpeg + MediaCodec 做一个播放器，实现视频的硬解码和音视频同步。
Android FFmpeg + MediaCodec 实现视频硬解码
MediaCodec 介绍
MediaCodec 是 Android 提供的用于对音视频进行编解码的类，它通过访问底层的 codec 来实现编解码的功能，是 Android media 基础框架的一部分，通常和 MediaExtractor, MediaSync, MediaMuxer, MediaCrypto, MediaDrm, Image, Surface和AudioTrack 一起使用。
mediacodec_buffers
详细描述可参见官方文档：https://developer.android.com/reference/android/media/MediaCodec.html
AMediaCodec 是 MediaCodec 的 native 接口，Google 从 Android 5.0 开始提供，Native 代码编译时需要引入 mediandk 库，官方 demo :
https://github.com/android/ndk-samples/tree/main/native-codec
FFmpeg + ANativeCodec
在 Android 没有在 Native 层开放 ModecCodec 接口之前，FFmpeg 实现硬解码需要将视频和音频数据拷贝到 Java 层，在 Java 层调用 MediaCodec (通过 JNI 调用 Java 对象方法)。
本文将实现 FFmpeg 和 AMediaCodec 结合使用， FFmpeg 负责解复用和音频解码，MediaCodec 负责视频解码并输出到 Surface（ANativeWindow）对象，其中解复用、音频解码、视频解码分别在一个子线程进行，利用队列管理音视频数据包。
注意：本 demo 处理的视频是 H.264 编码，由于 AVPacket data 不是标准的 NALU ，需要利用 av_bitstream_filter_filter 将 AVPacket 前 4 个字节替换为 0x00000001 得到标准的 NALU 数据，这样保证 MediaCodec 解码正确。
配置 AMediaCodec 对象，只解码视频流：
m_MediaExtractor = AMediaExtractor_new();
media_status_t err = AMediaExtractor_setDataSourceFd(m_MediaExtractor, fd,static_cast(outStart),static_cast(outLen));
close(fd);
if (err != AMEDIA_OK) {
result = -1;
LOGCATE("HWCodecPlayer::InitDecoder AMediaExtractor_setDataSourceFd fail. err=%d", err);
break;
}
int numTracks = AMediaExtractor_getTrackCount(m_MediaExtractor);
LOGCATE("HWCodecPlayer::InitDecoder AMediaExtractor_getTrackCount %d tracks", numTracks);
for (int i = 0; i < numTracks; i++) {
AMediaFormat *format = AMediaExtractor_getTrackFormat(m_MediaExtractor, i);
const char *s = AMediaFormat_toString(format);
LOGCATE("HWCodecPlayer::InitDecoder track %d format: %s", i, s);
const char *mime;
if (!AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mime)) {
LOGCATE("HWCodecPlayer::InitDecoder no mime type");
result = -1;
break;
} else if (!strncmp(mime, "video/", 6)) {
// Omitting most error handling for clarity.
// Production code should check for errors.
AMediaExtractor_selectTrack(m_MediaExtractor, i);
m_MediaCodec = AMediaCodec_createDecoderByType(mime);
AMediaCodec_configure(m_MediaCodec, format, m_ANativeWindow, NULL, 0);
AMediaCodec_start(m_MediaCodec);
}
AMediaFormat_delete(format);
}
FFmpeg 在一个线程中解复用，分别将音频和视频的编码数据包放入 2 个队列。
int HWCodecPlayer::DoMuxLoop() {
LOGCATE("HWCodecPlayer::DoMuxLoop start");
int result = 0;
AVPacket avPacket = {0};
for(;;) {
double passTimes = 0;
......
if(m_SeekPosition >= 0) { // seek 操作
//seek to frame
LOGCATE("HWCodecPlayer::DoMuxLoop seeking m_SeekPosition=%f", m_SeekPosition);
......
}
result = av_read_frame(m_AVFormatContext, &avPacket);
if(result >= 0) {
double bufferDuration = m_VideoPacketQueue->GetDuration() * av_q2d(m_VideoTimeBase);
LOGCATE("HWCodecPlayer::DoMuxLoop bufferDuration=%lfs", bufferDuration);
//防止缓冲数据包过多
while (BUFF_MAX_VIDEO_DURATION < bufferDuration && m_PlayerState == PLAYER_STATE_PLAYING && m_SeekPosition < 0) {
bufferDuration = m_VideoPacketQueue->GetDuration() * av_q2d(m_VideoTimeBase);
usleep(10 * 1000);
}
//音频和视频的编码数据包分别放入 2 个队列。
if(avPacket.stream_index == m_VideoStreamIdx) {
m_VideoPacketQueue->PushPacket(&avPacket);
} else if(avPacket.stream_index == m_AudioStreamIdx) {
m_AudioPacketQueue->PushPacket(&avPacket);
} else {
av_packet_unref(&avPacket);
}
} else {
//解复用结束，暂停解码器
std::unique_lock lock(m_Mutex);
m_PlayerState = PLAYER_STATE_PAUSE;
}
}
LOGCATE("HWCodecPlayer::DoMuxLoop end");
return 0;
}
视频解码线程中，Native 使用 AMediaCodec 对视频进行解码，从视频的 AVPacket 队列中取包进行解码。
void HWCodecPlayer::VideoDecodeThreadProc(HWCodecPlayer *player) {
LOGCATE("HWCodecPlayer::VideoDecodeThreadProc start");
AVPacketQueue* videoPacketQueue = player->m_VideoPacketQueue;
AMediaCodec* videoCodec = player->m_MediaCodec;
AVPacket *packet = av_packet_alloc();
for(;;) {
....
ssize_t bufIdx = -1;
bufIdx = AMediaCodec_dequeueInputBuffer(videoCodec, 0);
if (bufIdx >= 0) {
size_t bufSize;
auto buf = AMediaCodec_getInputBuffer(videoCodec, bufIdx, &bufSize);
av_bitstream_filter_filter(player->m_Bsfc, player->m_VideoCodecCtx, NULL, &packet->data, &packet->size, packet->data, packet->size,
packet->flags & AV_PKT_FLAG_KEY);
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc 0x%02X 0x%02X 0x%02X 0x%02X \n",packet->data[0],packet->data[1],packet->data[2],packet->data[3]);
memcpy(buf, packet->data, packet->size);
AMediaCodec_queueInputBuffer(videoCodec, bufIdx, 0, packet->size, packet->pts, 0);
}
av_packet_unref(packet);
AMediaCodecBufferInfo info;
auto status = AMediaCodec_dequeueOutputBuffer(videoCodec, &info, 1000);
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc status: %d\n", status);
uint8_t* buffer;
if (status >= 0) {
SyncClock* videoClock = &player->m_VideoClock;
double presentationNano = info.presentationTimeUs * av_q2d(player->m_VideoTimeBase) * 1000;
videoClock->SetClock(presentationNano, GetSysCurrentTime());
player->AVSync();//音视频同步
size_t size;
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc sync video curPts = %lf", presentationNano);
buffer = AMediaCodec_getOutputBuffer(videoCodec, status, &size);
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc buffer: %p, buffer size: %d", buffer, size);
AMediaCodec_releaseOutputBuffer(videoCodec, status, info.size != 0);
} else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc output buffers changed");
} else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc output format changed");
} else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc no output buffer right now");
} else {
LOGCATI("HWCodecPlayer::VideoDecodeThreadProc unexpected info code: %zd", status);
}
if(isLocked) lock.unlock();
}
if(packet != nullptr) {
av_packet_free(&packet);
packet = nullptr;
}
LOGCATE("HWCodecPlayer::VideoDecodeThreadProc end");
}
视频向音频同步，delay 时间根据实际帧率与目标帧率的时间差进行微调。
void HWCodecPlayer::AVSync() {
LOGCATE("HWCodecPlayer::AVSync");
double delay = m_VideoClock.curPts - m_VideoClock.lastPts;
int tickFrame = 1000 * m_FrameRate.den / m_FrameRate.num;
LOGCATE("HWCodecPlayer::AVSync tickFrame=%dms", tickFrame);
if(delay <= 0 || delay > VIDEO_FRAME_MAX_DELAY) {
delay = tickFrame;
}
double refClock = m_AudioClock.GetClock();// 视频向音频同步
double avDiff = m_VideoClock.lastPts - refClock;
m_VideoClock.lastPts = m_VideoClock.curPts;
double syncThreshold = FFMAX(AV_SYNC_THRESHOLD_MIN, FFMIN(AV_SYNC_THRESHOLD_MAX, delay));
LOGCATE("HWCodecPlayer::AVSync refClock=%lf, delay=%lf, avDiff=%lf, syncThreshold=%lf", refClock, delay, avDiff, syncThreshold);
if(avDiff <= -syncThreshold) { //视频比音频慢
delay = FFMAX(0, delay + avDiff);
}
else if(avDiff >= syncThreshold && delay > AV_SYNC_FRAMEDUP_THRESHOLD) { //视频比音频快太多
delay = delay + avDiff;
}
else if(avDiff >= syncThreshold)
delay = 2 * delay;
LOGCATE("HWCodecPlayer::AVSync avDiff=%lf, delay=%lf", avDiff, delay);
double tickCur = GetSysCurrentTime();
double tickDiff = tickCur - m_VideoClock.frameTimer;//两帧实际的时间间隔
m_VideoClock.frameTimer = tickCur;
if(tickDiff - tickFrame > 5) delay-=5;//微调delay时间
if(tickDiff - tickFrame < -5) delay+=5;
LOGCATE("HWCodecPlayer::AVSync delay=%lf, tickDiff=%lf", delay, tickDiff);
if(delay > 0) {
usleep(1000 * delay);
}
}
实现代码路径：
https://github.com/githubhaohao/LearnFFmpeg