这篇文章是对之前几篇文章的总结，实现了音频和视频的同步播放。
先看下流程图和类图
同步流程
类图
先说下几个类的作用：
1.FFmpegPlayer
与 jni 相关的类，主要功能是接收 java 层的命令，主要有相关类的初始化，接收 java 视频的播放，暂停，跳转功能并将这些命令转发到视频通道和音频通道等等。
2.BaseChannel
解码播放视频类和音频类的父类（相当于一个接口），主要是定义一些视频和音频共同的属性和函数，比如属性有java的回调类，时间基，package队列，frame队列，通道id，是否在播放等等属性，方法的话有暂停，播放，释放，跳转，释放package队列和frame队列等等。
3.AudioChannel
音频的实现类，继承与 BaseChannel，实现了 BaseChannel 中定义的方法及一些音频独有的方法等，比如 open sles 相关等。
4.VideoChannel
视频的实现类，继承与 BaseChannel，实现了 BaseChannel 中定义的方法及一些视频独有的方法等，比如画面渲染相关等。
5.LockQueue
是一个自定义的队列，是 package 和 frame 的队列，有添加，取出，释放，设置是否在工作等等方法
6.JavaCallHelper
给 java 层回调 错误信息，视频信息，播放的时间等等。
来讲下同步后播放的大概流程
1.首先是ffmpeg初始化后解析得到视频和音频流的index，得到音频和视频流的index后，新建音频和视频的解码播放类。
2.之后通过ffmpeg对视频解码后得到一个个package，读package信息将其送入相关的视频或音频的package队列，并且在这里做第一次的解码控制，控制 package 队列的长度为100，大于100就先堵塞一段时间，避免播放速度太慢，package队列有太多数据导致内存过高。
3.在视频和音频类中将 package 队列中的 package数据取出解码出一个个的 frame 帧数据并将其送入 frame 队列，在fame队列也做了和package队列解码控制个数的操作。
4.在音频类中。先进行 open sles 的初始化，在拿到 frame 中的数据并通过 ffmpeg 进行重采样等处理将其送入 open sles 进行音频播放并记录播放的时间以便于和视频做同步处理。
5.在视频类中，先拿到 frame 中的数据，通过 ffmpeg 将其转换成统一输出的格式，将 ffmpeg 转换后的数据送入 ANativeWindow 进行渲染。
6.音视频同步。视频类在送入渲染后拿到该 frame 帧的时间和音频播放的是时间做比较，在相差较大时，如果视频时间在前则睡眠一段时间，反之就丢掉 frame 队列中前面一帧的数据已达到视频和音频时间的同步。
void FFmpegPlayer::prepare() {
javaCallHelper->call_java_status(THREAD_MAIN, PLAYER_PREPARE);
pthread_create(&prepare_pid, NULL, pthread_prepare, this);
pthread_detach(prepare_pid);
}
void FFmpegPlayer::prepareFFmpeg() {
// 1. 注册组件，就是一些初始化工作
avformat_network_init();
this->formatContext = avformat_alloc_context();
//初始化 ffmpeg 参数
AVDictionary *options = NULL;
int ret = -1;
if (av_dict_set(&options, "timeout", "3000000", 0) < 0) {
error(AV_DICT_SET_ERROR, "av_dict_set error");
LOGE("av_dict_set error");
return;
}
//NULL 输入文件的封装格式,FFmpeg自动检测 AVInputFormat
// 2. 打开视频文件
if ((ret = avformat_open_input(&this->formatContext, this->url, NULL, &options)) != 0) {
char buf[1024];
av_strerror(ret, buf, 1024);
error(FFMPEG_CAN_NOT_OPEN_URL, "avformat_open_input error");
LOGE("avformat_open_input error Couldn’t open file %s: %d(%s)", this->url, ret, buf);
return;
}
// 3. 获取视频信息
if (avformat_find_stream_info(this->formatContext, NULL) < 0) {
error(FFMPEG_CAN_NOT_FIND_STREAMS, "avformat_find_stream_info error");
LOGE("avformat_find_stream_info error");
return;
}
// 4. 查找视频流
for (int i = 0; i < this->formatContext->nb_streams; ++i) {
AVCodecParameters *avCodecParameters = this->formatContext->streams[i]->codecpar;
AVCodec *avCodec = avcodec_find_decoder(avCodecParameters->codec_id);
AVCodecContext *avCodecContext = avcodec_alloc_context3(avCodec);
if (avcodec_parameters_to_context(avCodecContext, avCodecParameters) < 0) {
LOGE("avcodec_parameters_to_context error");
return;
}
if (avcodec_open2(avCodecContext, avCodec, NULL) != 0) {
LOGE("avcodec_open2 error");
return;
}
if (this->formatContext->streams[i]->codecpar->codec_type == AVMediaType::AVMEDIA_TYPE_VIDEO) {
if (!videoChannel) {
LOGE("videoChannel channel id is %d", i);
videoChannel = new VideoChannel(avCodecContext, this->formatContext->streams[i]->time_base, i,
this->javaCallHelper);
videoChannel->setFrameRender(renderFrame);
AVRational frame_rate = this->formatContext->streams[i]->avg_frame_rate;
int fps = av_q2d(frame_rate);
videoChannel->setFps(fps);
}
} else if (this->formatContext->streams[i]->codecpar->codec_type == AVMediaType::AVMEDIA_TYPE_AUDIO) {
if (!audioChannel) {
LOGE("audioChannel channel id is %d", i);
audioChannel = new AudioChannel(avCodecContext, this->formatContext->streams[i]->time_base, i,
this->javaCallHelper);
}
}
}
if (!videoChannel && !audioChannel) {
return;
}
if (videoChannel && audioChannel) {
videoChannel->setAudioChannel(audioChannel);
javaCallHelper->call_java_ready(THREAD_CHILD, formatContext->duration / AV_TIME_BASE);
}
}
对应上面的流程1。因为 ffmpeg 初始化是耗时操作，所以开了一个线程
void FFmpegPlayer::start() {
this->isPlaying = true;
if (audioChannel) {
audioChannel->play();
}
if (videoChannel) {
videoChannel->play();
}
pthread_create(&play_pid, NULL, pthread_ffmpeg_play, this);
pthread_detach(play_pid);
javaCallHelper->call_java_status(THREAD_MAIN, PLAYER_PLAYING);
}
void FFmpegPlayer::play() {
int ret = 0;
while (this->isPlaying) {
if (videoChannel && videoChannel->package_queue.size() > 100) {
av_usleep(1000 * 10);
continue;
}
if (audioChannel && audioChannel->package_queue.size() > 100) {
av_usleep(1000 * 10);
continue;
}
AVPacket *avPacket = av_packet_alloc();
ret = av_read_frame(this->formatContext, avPacket);
// LOGE("avPacket->stream_index is %d", avPacket->stream_index);
if (ret == 0) {
if (avPacket->stream_index == videoChannel->channelId) {
videoChannel->package_queue.push(avPacket);
// LOGE("videoChannel package_queue push size is %d", videoChannel->package_queue.size());
} else if (avPacket->stream_index == audioChannel->channelId) {
audioChannel->package_queue.push(avPacket);
// LOGE("audioChannel package_queue push size is %d", audioChannel->package_queue.size());
}
} else if (ret == AVERROR_EOF) {
//读取完毕 但是不一定播放完毕
if (videoChannel->package_queue.empty() && videoChannel->frame_queue.empty() &&
audioChannel->package_queue.empty() && audioChannel->frame_queue.empty()) {
LOGE("播放完毕。。。");
break;
}
//因为seek 的存在，就算读取完毕，依然要循环 去执行av_read_frame(否则seek了没用...)
} else {
break;
}
}
//1.播放暂停
//2.播放完了
if (!isPause) {
stop();
}
LOGE("FFmpegPlayer play thread end");
}
对应上面流程2
void VideoChannel::play() {
this->package_queue.setWork(true);
this->frame_queue.setWork(true);
this->isPlaying = true;
pthread_create(&decode_pid, NULL, pthread_video_decode, this);
pthread_create(&play_pid, NULL, pthread_video_play, this);
pthread_detach(play_pid);
pthread_detach(decode_pid);
}
void *pthread_video_decode(void *context) {
VideoChannel *videoChannel = static_cast(context);
videoChannel->decodeVideoPacket();
return 0;
}
void VideoChannel::decodeVideoPacket() {
int ret;
AVPacket *packet = 0;
while (this->isPlaying) {
ret = package_queue.pop(packet);
// LOGE("VideoChannel package_queue pop size is %d", package_queue.size());
if (!isPlaying) {
break;
}
if (!ret) {
continue;
}
avcodec_send_packet(avCodecContext, packet);
releaseAvPacket(packet);
AVFrame *avFrame = av_frame_alloc();
ret = avcodec_receive_frame(avCodecContext, avFrame);
if (ret == AVERROR(EAGAIN)) {
continue;
} else if ((ret == AVERROR(EINVAL)) || (ret == AVERROR_EOF)) {
break;
} else if (ret < 0) {
LOGE("legitimate decoding 2 errors");
break;
}
frame_queue.push(avFrame);
// LOGE("VideoChannel frame_queue push size is %d", frame_queue.size());
while (frame_queue.size() > 100 && isPlaying) {
av_usleep(1000 * 10);
continue;
}
}
releaseAvPacket(packet);
LOGE("VideoChannel decodeVideoPacket thread end");
}
void AudioChannel::play() {
swrContext = swr_alloc_set_opts(0,
AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_S16, out_sample_rate,
avCodecContext->channel_layout,
avCodecContext->sample_fmt,
avCodecContext->sample_rate, 0, 0);
swr_init(swrContext);
if (swr_init(swrContext) < 0) {
LOGE("swr_init error");
return;
}
this->package_queue.setWork(true);
this->frame_queue.setWork(true);
this->isPlaying = true;
pthread_create(&decode_pid, NULL, pthread_audio_decode, this);
pthread_create(&play_pid, NULL, pthread_audio_play, this);
pthread_detach(play_pid);
pthread_detach(decode_pid);
}
void *pthread_audio_decode(void *context) {
AudioChannel *audioChannel = static_cast(context);
audioChannel->decodeAudioPacket();
return 0;
}
void AudioChannel::decodeAudioPacket() {
int ret;
AVPacket *packet = 0;
while (this->isPlaying) {
ret = package_queue.pop(packet);
// LOGE("AudioChannel package_queue pop size is %d", package_queue.size());
if (!isPlaying) {
break;
}
if (!ret) {
continue;
}
ret = avcodec_send_packet(avCodecContext, packet);
releaseAvPacket(packet);
if (ret == AVERROR(EAGAIN)) {
//需要更多数据
continue;
} else if (ret < 0) {
//失败
LOGE("AudioChannel avcodec_send_packet errors");
break;
}
AVFrame *avFrame = av_frame_alloc();
ret = avcodec_receive_frame(avCodecContext, avFrame);
if (ret == AVERROR(EAGAIN)) {
continue;
} else if ((ret == AVERROR(EINVAL)) || (ret == AVERROR_EOF)) {
break;
} else if (ret < 0) {
LOGE("AudioChannel legitimate decoding 2 errors");
break;
}
while (frame_queue.size() > 100 && isPlaying) {
av_usleep(1000 * 10);
continue;
}
frame_queue.push(avFrame);
// LOGE("AudioChannel frame_queue push size is %d", frame_queue.size());
}
// releaseAvPacket(packet);
LOGE("AudioChannel decodeAudioPacket thread end");
}
这里对应流程3，视频和音频解码 package 并送入 frame 队列中。
void *pthread_audio_play(void *context) {
AudioChannel *audioChannel = static_cast(context);
audioChannel->init_opensl_es();
return 0;
}
void AudioChannel::init_opensl_es() {
//执行结果
SLresult result;
//引擎对象接口
SLObjectItf engineObj;
//引擎对象实例
SLEngineItf engineItf;
//1.创建引擎 三步走
result = slCreateEngine(&engineObj, 0, 0, 0, 0, 0);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*engineObj)->Realize(engineObj, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*engineObj)->GetInterface(engineObj, SL_IID_ENGINE, &engineItf);
assert(SL_RESULT_SUCCESS == result);
(void) result;
//2.创建混音器 这些参数大多都是从 google simple 中拿的，具体的含义不太清楚
SLObjectItf mixerObj;
SLEnvironmentalReverbItf outputMixEnvironmentalReverb = NULL;
SLEnvironmentalReverbSettings reverbSettings = SL_I3DL2_ENVIRONMENT_PRESET_STONECORRIDOR;
const SLInterfaceID ids[1] = {SL_IID_ENVIRONMENTALREVERB};
const SLboolean req[1] = {SL_BOOLEAN_FALSE};
result = (*engineItf)->CreateOutputMix(engineItf, &mixerObj, 1, ids, req);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*mixerObj)->Realize(mixerObj, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*mixerObj)->GetInterface(mixerObj, SL_IID_ENVIRONMENTALREVERB,
&outputMixEnvironmentalReverb);
if (SL_RESULT_SUCCESS == result) {
result = (*outputMixEnvironmentalReverb)->SetEnvironmentalReverbProperties(
outputMixEnvironmentalReverb, &reverbSettings);
(void) result;
}
//3.创建播放器
SLObjectItf playerObj;
SLDataLocator_AndroidSimpleBufferQueue android_queue = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
2};
SLDataFormat_PCM pcm = {
SL_DATAFORMAT_PCM,//播放pcm格式的数据
2,//2个声道（立体声）
SL_SAMPLINGRATE_44_1,//44100hz的频率
SL_PCMSAMPLEFORMAT_FIXED_16,//位数 16位
SL_PCMSAMPLEFORMAT_FIXED_16,//和位数一致就行
SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT,//立体声（前左前右）
SL_BYTEORDER_LITTLEENDIAN//结束标志
};
SLDataSource pAudioSrc = {&android_queue, &pcm};
SLDataLocator_OutputMix loc_outmix = {SL_DATALOCATOR_OUTPUTMIX, mixerObj};
SLDataSink pAudioSnk = {&loc_outmix, NULL};
SLuint32 numInterfaces = 3;
const SLInterfaceID pInterfaceIds[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND,
/*SL_IID_MUTESOLO,*/};
const SLboolean pInterfaceRequired[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
/*SL_BOOLEAN_TRUE,*/ };
result = (*engineItf)->CreateAudioPlayer(engineItf, &playerObj, &pAudioSrc, &pAudioSnk,
numInterfaces,
pInterfaceIds, pInterfaceRequired);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*playerObj)->Realize(playerObj, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*playerObj)->GetInterface(playerObj, SL_IID_PLAY, &playItf);
assert(SL_RESULT_SUCCESS == result);
(void) result;
//4.设置回调队列及回调函数
SLVolumeItf pcmPlayerVolume = NULL;
result = (*playerObj)->GetInterface(playerObj, SL_IID_BUFFERQUEUE, &bqPlayerBufferQueue);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*bqPlayerBufferQueue)->RegisterCallback(bqPlayerBufferQueue, audioChannelBufferCallBack, this);
assert(SL_RESULT_SUCCESS == result);
(void) result;
//获取音量接口
result = (*playerObj)->GetInterface(playerObj, SL_IID_VOLUME, &pcmPlayerVolume);
assert(SL_RESULT_SUCCESS == result);
(void) result;
//5.设置播放状态
(*playItf)->SetPlayState(playItf, SL_PLAYSTATE_PLAYING);
//6.回调
audioChannelBufferCallBack(bqPlayerBufferQueue, this);
LOGE("--- 手动调用播放 packet:%d", this->package_queue.size());
}
int AudioChannel::getPackageSize() {
int data_size = 0;
int ret;
AVFrame *avFrame = 0;
while (isPlaying) {
ret = frame_queue.pop(avFrame);
// LOGE("AudioChannel frame_queue pop size is %d", frame_queue.size());
if (!isPlaying) {
break;
}
if (!ret) {
continue;
}
uint64_t dst_nb_samples = av_rescale_rnd(
swr_get_delay(swrContext, avFrame->sample_rate) + avFrame->nb_samples,
out_sample_rate,
avFrame->sample_rate,
AV_ROUND_UP);
// 转换，返回值为转换后的sample个数 buffer malloc（size）
int nb = swr_convert(swrContext, &buffer, dst_nb_samples,
(const uint8_t **) avFrame->data, avFrame->nb_samples);
//判断这个是因为之前我直接通过 swr_alloc_set_opts() 拿到了 SwrContext， 之后没有对 SwrContext 进行初始化 swr_init() 在 swr_convert() 报了一个 Invalid argument的错误
if (nb < 0) {
char buf[1024];
av_strerror(ret, buf, 1024);
LOGE("swr_convert error %d(%s)", ret, buf);//Invalid argument
continue;
}
// int out_channels = av_get_channel_layout_nb_channels(AV_CH_LAYOUT_STEREO);
// data_size = ret * out_channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);
// //转换后多少数据 buffer size 44110*2*2
data_size = nb * out_channels * out_samplesize;
clock = avFrame->pts * av_q2d(time_base);
// LOGE("解码一帧音频 %d,clock is %f,last_time is %f", frame_queue.size(), clock, last_time);
if (abs(clock - last_time) > 1.0) {
last_time = clock;
javaCallHelper->call_java_videoInfo(THREAD_CHILD, 60, static_cast(clock));
}
break;
}
releaseAvFrame(avFrame);
return data_size;
}
void audioChannelBufferCallBack(SLAndroidSimpleBufferQueueItf bf, void *context) {
//assert(NULL == context);
AudioChannel *audioChannel = static_cast(context);
int size = audioChannel->getPackageSize();
if (size > 0) {
// LOGE("size is %d", size);
SLresult result;
// enqueue another buffer
result = (*bf)->Enqueue(bf, audioChannel->buffer, size);
} else {
LOGE("size null");
}
}
对应流程4
void *pthread_video_play(void *context) {
VideoChannel *videoChannel = static_cast(context);
videoChannel->render();
return 0;
}
void VideoChannel::render() {
//解码上下文 指定播放的是 argb8888 格式 ，宽高等属性和输入视频一致
SwsContext *sws_ctx = sws_getContext(
avCodecContext->width, avCodecContext->height, avCodecContext->pix_fmt,
avCodecContext->width, avCodecContext->height, AV_PIX_FMT_RGBA,
SWS_BILINEAR, 0, 0, 0);
uint8_t *dst_data[4];
int dst_linesize[4];
// 一帧视频的大小
av_image_alloc(dst_data, dst_linesize,
avCodecContext->width, avCodecContext->height,
AV_PIX_FMT_RGBA, 1);
int ret;
AVFrame *avFrame = 0;
while (isPlaying) {
ret = frame_queue.pop(avFrame);
// LOGE("VideoChannel frame_queue pop size is %d", frame_queue.size());
if (!isPlaying) {
break;
}
if (!ret) {
continue;
}
// 数据：avFrame -> dst_data
sws_scale(sws_ctx,
reinterpret_cast(avFrame->data), avFrame->linesize, 0,
avFrame->height,
dst_data, dst_linesize);
renderFrame(avCodecContext->width, avCodecContext->height, dst_data[0], dst_linesize[0]);
// LOGE("解码一帧视频 %d", frame_queue.size());
clock = avFrame->pts * av_q2d(time_base);
//延时的来源 解码延时 播放延时
//解码时间 看注释 extra_delay = repeat_pict / (2*fps)
double delay = avFrame->repeat_pict / (2 * fps);
double audioClock = audioChannel->clock;
double diff = clock - audioClock - fastTime;
// LOGE("diff is %f,delay is %f", diff, delay);
if (clock > audioClock) {//视频在前
if (diff > 1) {//差的太多，睡双倍
av_usleep((delay * 2) * 1000000);
} else {//差的不多，延时
av_usleep((delay + diff) * 1000000);
}
} else {//音频在前
if (diff > 1) {//不休眠
} else if (diff >= 0.05) {//差的不多 视频需要丢包 同步
releaseAvFrame(avFrame);
AVFrame *frame;
if (frame_queue.pop(frame)) {
releaseAvFrame(frame);
}
} else { //差不多 可以选择性的丢包 删除 key frame
}
}
releaseAvFrame(avFrame);
}
//播放完了
av_freep(&dst_data[0]);
isPlaying = false;
releaseAvFrame(avFrame);
sws_freeContext(sws_ctx);
LOGE("VideoChannel render thread end");
}
对应流程5和流程6。
至此音频和视频就可以同步的播放。