FFMPEG+SDL2 实现播放器功能
ffmpeg视频H264压缩，rtsp推流课程教学视频：
https://edu.csdn.net/course/detail/27795
课件中里面提供源码可以直接下载运行！
-------------------------------------------------------------------------------------------------------------------------------------
一、界面
1 打开的pushbutton
2 停止/开始的pushbutton
3 进度条 QSlider
4 播放窗口widget
二、创建videothread线程
该线程用于读取视频数据流 具体实现过程可以参考
http://blog.csdn.net/yunge812/article/details/79342089
其中需要添加SDL处理音频的部分进去
2.1 SDL初始化
if (SDL_Init(SDL_INIT_AUDIO))
{
fprintf(stderr,"Could not initialize SDL - %s. \n", SDL_GetError());
exit(1);
}
2.2 获取音频流信息
for (i = 0; i < pFormatCtx->nb_streams; i++)
{
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0)
{
audioStream = i;
}
}
///如果audioStream为-1 说明没有找到音频流
if (audioStream == -1) {
printf("Didn't find a audio stream.\n");
return;
}
if (audioStream >= 0)
{
/// 设置SDL音频流信息 音频处理初始化
audio_stream_component_open(&mVideoState, audioStream); //该函数后续会进行完成
}
audio_stream_component_open函数在3.3.1中会涉及
2.3 查找音频解码器
///================================ 查找音频解码器 =================================//
AVCodecContext *aCodecCtx;
aCodecCtx = pFormatCtx->streams[audioStream]->codec;
AVCodec *aCodec;
aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
if (aCodec == NULL)
{
printf("ACodec not found.\n");
return;
}
///================================ 打开音频解码器 ==================================//
if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
{
printf("Could not open audio codec.\n");
return;
} 1 SDL初始化
vs->audio_st = pFormatCtx->streams[audioStream]; ///设置音频解码
packet_queue_init(&vs->videoq); //音频处理：初始化队列2.4 创建视频解码的线程 由SDL进行创建 video_thread函数后续阶段完成
vs->video_tid = SDL_CreateThread(video_thread, "video_thread", &mVideoState);
vs->player = this; //线程指针
2.5 分配AVPacket用于存放读取的视频
AVPacket *packet = (AVPacket *) malloc(sizeof(AVPacket)); //分配一个packet 用于存放读取的视频三、开始进行音视频的读取
3.1 音视频统一读取
//=========================== 读取视频信息 ===============================//
if (av_read_frame(pFormatCtx, packet) < 0) //读取的是一帧视频 数据存入一个AVPacket的结构中
{
qDebug() << "read error." ;
return ;
}
//此时数据存储在packet中
if (packet->stream_index == videoStream)
{
packet_queue_put(&vs->videoq, packet); //这里我们将视频数据存入队列
}
else if( packet->stream_index == audioStream )
{
packet_queue_put(&vs->audioq, packet);//这里我们将音频数据存入队列
}3.2 视频读取线程 将视频转换成QImage显示
3.2.1 视频相关结构体的分配
///解码视频相关
AVFrame *pFrame, *pFrameRGB;
uint8_t *out_buffer_rgb; //解码后的rgb数据
struct SwsContext *img_convert_ctx; //用于解码后的视频格式转换
AVCodecContext *pCodecCtx = is->video_st->codec; //视频解码器
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
///这里我们改成了 将解码后的YUV数据转换成RGB32
img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,
pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,
PIX_FMT_RGB32, SWS_BICUBIC, NULL, NULL, NULL);
numBytes = avpicture_get_size(PIX_FMT_RGB32, pCodecCtx->width,pCodecCtx->height);
out_buffer_rgb = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
avpicture_fill((AVPicture *) pFrameRGB, out_buffer_rgb, PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);
// 会将pFrameRGB的数据按RGB格式自动"关联"到out_buffer_rgb 即pFrameRGB中的数据改变了 out_buffer中的数据也会相应的改变3.2.2 从队列中取出数据
if (packet_queue_get(&is->videoq, packet, 1) <= 0)
break;//队列里面没有数据了 读取完毕了
3.2.3 解码
ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture,packet);
if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE)
{
video_pts = *(uint64_t *) pFrame->opaque;
}
else if (packet->dts != AV_NOPTS_VALUE)
{
video_pts = packet->dts;
}
else
{
video_pts = 0;
}
3.2.4 音视频同步 关键函数synchronize_video( )
video_pts *= av_q2d(is->video_st->time_base);
video_pts = synchronize_video(is, pFrame, video_pts);
while(1)
{
audio_pts = is->audio_clock;
if (video_pts <= audio_pts) //等待匹配 两者时间匹配 跳出该循环
break;
int delayTime = (video_pts - audio_pts) * 1000;
delayTime = delayTime > 5 ? 5:delayTime;
SDL_Delay(delayTime);
}
3.2.5 将图像缩放 转换格式 输出到QImage 并且激活信号
if (got_picture)
{
sws_scale(img_convert_ctx,(uint8_t const * const *) pFrame->data,
pFrame->linesize, 0, pCodecCtx->height, pFrameRGB->data,
pFrameRGB->linesize);
//把这个RGB数据 用QImage加载
QImage tmpImg((uchar *)out_buffer_rgb,pCodecCtx->width,pCodecCtx->height,QImage::Format_RGB32);
QImage image = tmpImg.copy(); //把图像复制一份 传递给界面显示
is->player->disPlayVideo(image); //调用激发信号的函数
}
3.3 音频处理过程
3.3.1 audio_stream_component_open 函数
int audio_stream_component_open(VideoState *is, int stream_index) //audioStream 遍历出来的音频流信息
{
AVFormatContext *ic = is->ic;
AVCodecContext *codecCtx;
AVCodec *codec;
SDL_AudioSpec wanted_spec, spec; // wanted_spec是我们期望设置的属性，spec是系统最终接受的参数
int64_t wanted_channel_layout = 0;
int wanted_nb_channels; //声道数
/* SDL支持的声道数为 1, 2, 4, 6 */
/* 后面我们会使用这个数组来纠正不支持的声道数目 */
const int next_nb_channels[] = { 0, 0, 1, 6, 2, 6, 4, 6 };
if (stream_index < 0 || stream_index >= ic->nb_streams)
{
return -1;
}
codecCtx = ic->streams[stream_index]->codec;
wanted_nb_channels = codecCtx->channels;
//根据期望的声道数获得期望的声道布局
if (!wanted_channel_layout || wanted_nb_channels!= av_get_channel_layout_nb_channels(wanted_channel_layout)) //Return the number of channels in the channel layout.
{
wanted_channel_layout = av_get_default_channel_layout(wanted_nb_channels); //Return default channel layout for a given number of channels.
wanted_channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
}
wanted_spec.channels = av_get_channel_layout_nb_channels(wanted_channel_layout);//根据期望的声道布局获得期望的声道数
wanted_spec.freq = codecCtx->sample_rate; //频率 由外部传入流转换得到
if (wanted_spec.freq <= 0 || wanted_spec.channels <= 0)
{
return -1;
}
//设置期望属性
wanted_spec.format = AUDIO_S16SYS; // 格式
wanted_spec.silence = 0; // 0指示静音
wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE; // 自定义SDL缓冲区大小
wanted_spec.callback = audio_callback; // 音频解码的关键回调函数
wanted_spec.userdata = is; // 传给上面回调函数的外带数据
//打开音频设备，这里使用一个while来循环尝试打开不同的声道数(由上面next_nb_channels数组指定)直到成功打开，或者全部失败
//wanted_spec ==> spec
do
{
is->audioID = SDL_OpenAudioDevice(SDL_GetAudioDeviceName(0,0),0,&wanted_spec, &spec,0);
//is->audioID = SDL_OpenAudio(&wanted_spec, &spec);
fprintf(stderr,"SDL_OpenAudio (%d channels): %s\n",wanted_spec.channels, SDL_GetError());
qDebug()<audioID == 0);
// 检查实际使用的配置（保存在spec,由SDL_OpenAudio()填充）
if (spec.format != AUDIO_S16SYS)
{
fprintf(stderr,"SDL advised audio format %d is not supported!\n",spec.format);
return -1;
}
// 看二者的channels是否相同 得到最终的wanted_channel_layout 该参数需要传给VideoPlayer
if (spec.channels != wanted_spec.channels)
{
wanted_channel_layout = av_get_default_channel_layout(spec.channels);
if (!wanted_channel_layout)
{
fprintf(stderr,"SDL advised channel count %d is not supported!\n",spec.channels);
return -1;
}
}
is->audio_hw_buf_size = spec.size;
/* 把设置好的参数保存到大结构VideoState中 */
is->audio_src_fmt = is->audio_tgt_fmt = AV_SAMPLE_FMT_S16;
is->audio_src_freq = is->audio_tgt_freq = spec.freq;
is->audio_src_channel_layout = is->audio_tgt_channel_layout = wanted_channel_layout;
is->audio_src_channels = is->audio_tgt_channels = spec.channels;
///====================================== 查找解码器 ==========================================//
codec = avcodec_find_decoder(codecCtx->codec_id);
///====================================== 打开解码器 ==========================================//
if (!codec || (avcodec_open2(codecCtx, codec, NULL) < 0))
{
fprintf(stderr,"Unsupported codec!\n");
return -1;
}
ic->streams[stream_index]->discard = AVDISCARD_DEFAULT; //丢掉大小为0的数据流
switch (codecCtx->codec_type)
{
case AVMEDIA_TYPE_AUDIO:
is->audio_st = ic->streams[stream_index];
is->audio_buf_size = 0;
is->audio_buf_index = 0;
memset(&is->audio_pkt, 0, sizeof(is->audio_pkt));
packet_queue_init(&is->audioq); //初始化PacketQueue队列 全部清零
//SDL_PauseAudio(0); // 开始播放静音
SDL_PauseAudioDevice(is->audioID,0); //开始播放 调用回调函数audio_callback
break;
default:
break;
}
return 0;
}
该函数基本上是固定模式 需要注意的是回调函数的设置
wanted_spec.callback = audio_callback; // 音频解码的关键回调函数3.3.2 回调函数 在开始播放后调用audio_callback
static void audio_callback(void *userdata, Uint8 *stream, int len)
{
VideoState *is = (VideoState *) userdata; //传入的数据结构体
int len1, audio_data_size;
double pts;
/* len是由SDL传入的SDL缓冲区的大小，如果这个缓冲未满，我们就一直往里填充数据 */
/* audio_buf_index 和 audio_buf_size 标示我们自己用来放置解码出来的数据的缓冲区 */
/* 这些数据待copy到SDL缓冲区， 当audio_buf_index >= audio_buf_size的时候意味着我们的缓冲为空
/* 没有数据可供copy，这时候需要调用audio_decode_frame来解码出更多的桢数据 */
while (len > 0)
{
if (is->audio_buf_index >= is->audio_buf_size) //audio_buf为空 需要解码更多帧的数据
{
///解码之后的数据存储在 is->audio_buf
audio_data_size = audio_decode_frame(is, &pts); //解码的过程
if (audio_data_size < 0) /* 没能解码出数据，我们默认播放静音 */
{
/* silence */
is->audio_buf_size = 1024;
/* 清零，静音 */
memset(is->audio_buf, 0, is->audio_buf_size);
}
else
{
is->audio_buf_size = audio_data_size;
}
is->audio_buf_index = 0; //指针归零
}
/* 查看stream可用空间，决定一次copy多少数据，剩下的下次继续copy */
len1 = is->audio_buf_size - is->audio_buf_index;
if (len1 > len)
{
len1 = len;
}
// is->audio_buf ===> stream
memcpy(stream, (uint8_t *) is->audio_buf + is->audio_buf_index, len1);
len -= len1;
stream += len1;
is->audio_buf_index += len1;
}
//qDebug()<<"audio_callback finished";
}
3.3.3 解码的过程audio_decode_frame
static int audio_decode_frame(VideoState *is, double *pts_ptr) //解码音频
{
int len1, len2, decoded_data_size;
AVPacket *pkt = &is->audio_pkt;
int got_frame = 0;
int64_t dec_channel_layout;
int wanted_nb_samples, resampled_data_size, n;
double pts;
for (;;)
{
while (is->audio_pkt_size > 0)
{
//判断暂停
if (is->isPause == true)
{
SDL_Delay(10);
continue;
}
if (!is->audio_frame)
{
if (!(is->audio_frame = avcodec_alloc_frame()))
{
return AVERROR(ENOMEM);
}
}
else
avcodec_get_frame_defaults(is->audio_frame);
//解码 数据存储在is->audio_frame
len1 = avcodec_decode_audio4(is->audio_st->codec, is->audio_frame,&got_frame, pkt);
if (len1 < 0) // error, skip the frame
{
is->audio_pkt_size = 0;
break;
}
is->audio_pkt_data += len1; //指针+len1
is->audio_pkt_size -= len1; //缓存-len1
if (!got_frame)
continue;
/* 计算解码出来的桢需要的缓冲大小 */
decoded_data_size = av_samples_get_buffer_size(NULL,
is->audio_frame->channels, is->audio_frame->nb_samples,
(AVSampleFormat)is->audio_frame->format, 1);
dec_channel_layout =
(is->audio_frame->channel_layout && is->audio_frame->channels
== av_get_channel_layout_nb_channels( is->audio_frame->channel_layout))
?is->audio_frame->channel_layout :av_get_default_channel_layout(is->audio_frame->channels);
wanted_nb_samples = is->audio_frame->nb_samples;
if (is->audio_frame->format != is->audio_src_fmt
|| dec_channel_layout != is->audio_src_channel_layout
|| is->audio_frame->sample_rate != is->audio_src_freq
|| (wanted_nb_samples != is->audio_frame->nb_samples
&& !is->swr_ctx))
{
if (is->swr_ctx)
swr_free(&is->swr_ctx);
is->swr_ctx = swr_alloc_set_opts(NULL,
is->audio_tgt_channel_layout, (AVSampleFormat)is->audio_tgt_fmt,
is->audio_tgt_freq, dec_channel_layout,
(AVSampleFormat)is->audio_frame->format, is->audio_frame->sample_rate,
0, NULL);
if (!is->swr_ctx || swr_init(is->swr_ctx) < 0)
{
//fprintf(stderr,"swr_init() failed\n");
break;
}
is->audio_src_channel_layout = dec_channel_layout;
is->audio_src_channels = is->audio_st->codec->channels;
is->audio_src_freq = is->audio_st->codec->sample_rate;
is->audio_src_fmt = is->audio_st->codec->sample_fmt;
}
/* 这里我们可以对采样数进行调整，增加或者减少，一般可以用来做声画同步 */
if (is->swr_ctx)
{
const uint8_t **in =(const uint8_t **) is->audio_frame->extended_data;
uint8_t *out[] = { is->audio_buf2 };
if (wanted_nb_samples != is->audio_frame->nb_samples)
{
if (swr_set_compensation(is->swr_ctx,
(wanted_nb_samples - is->audio_frame->nb_samples)
* is->audio_tgt_freq
/ is->audio_frame->sample_rate,
wanted_nb_samples * is->audio_tgt_freq
/ is->audio_frame->sample_rate) < 0)
{
//fprintf(stderr,"swr_set_compensation() failed\n");
break;
}
}
//=================================== 转换 ===========================================//
//转换出来的数据存储在uint8_t *out中。也就是 is->audio_buf2
len2 = swr_convert(is->swr_ctx, out,
sizeof(is->audio_buf2) / is->audio_tgt_channels
/ av_get_bytes_per_sample(is->audio_tgt_fmt),
in, is->audio_frame->nb_samples);
if (len2 < 0)
{
//fprintf(stderr,"swr_convert() failed\n");
break;
}
if (len2 == sizeof(is->audio_buf2) / is->audio_tgt_channels
/ av_get_bytes_per_sample(is->audio_tgt_fmt))
{
//fprintf(stderr,"warning: audio buffer is probably too small\n");
swr_init(is->swr_ctx);
}
is->audio_buf = is->audio_buf2; //解码之后转换得到的数据
resampled_data_size = len2 * is->audio_tgt_channels
* av_get_bytes_per_sample(is->audio_tgt_fmt);
}
else
{
resampled_data_size = decoded_data_size;
is->audio_buf = is->audio_frame->data[0];
}
//计算时间
pts = is->audio_clock;
*pts_ptr = pts;
n = 2 * is->audio_st->codec->channels;
is->audio_clock += (double) resampled_data_size
/ (double) (n * is->audio_st->codec->sample_rate);
// We have data, return it and come back for more later
return resampled_data_size;
}
//判断暂停
if (is->isPause == true)
{
SDL_Delay(10);
continue;
}
if (pkt->data)
av_free_packet(pkt);
memset(pkt, 0, sizeof(*pkt));
if (packet_queue_get(&is->audioq, pkt, 0) <= 0) //从队列is->audioq中取出第一个packet 存放在pkt中
return -1;
is->audio_pkt_data = pkt->data;
is->audio_pkt_size = pkt->size;
/* if update, update the audio clock w/pts */
if (pkt->pts != AV_NOPTS_VALUE)
{
is->audio_clock = av_q2d(is->audio_st->time_base) * pkt->pts;
}
}
return 0;
}
3.4 synchronize_video( )音视频同步函数
static double synchronize_video(VideoState *is, AVFrame *src_frame, double pts)
{
double frame_delay;
if (pts != 0) {
/* if we have pts, set video clock to it */
is->video_clock = pts;
} else {
/* if we aren't given a pts, set it to the clock */
pts = is->video_clock;
}
/* update the video clock */
frame_delay = av_q2d(is->video_st->codec->time_base);
/* if we are repeating a frame, adjust clock accordingly */
frame_delay += src_frame->repeat_pict * (frame_delay * 0.5);
is->video_clock += frame_delay;
return pts;
}
总结：
总体来看系统由三个线程
1 音视频处理总线程 该线程由继承的QT类得到
2 视频处理线程 由SDL创建 vs->video_tid = SDL_CreateThread(video_thread, "video_thread", &mVideoState);
3 音频处理线程 由SDL创建 audio_stream_component_open( )中有音频数据的回调函数
同步过程 在线程2 视频处理线程中同步
/*===========================================音视频同步===============================================*/
video_pts = synchronize_video(is, pFrame, video_pts);
while(1)
{
audio_pts = is->audio_clock;
if (video_pts <= audio_pts) //等待匹配 两者时间匹配 跳出该循环
break;
int delayTime = (video_pts - audio_pts) * 1000;
delayTime = delayTime > 5 ? 5:delayTime;
SDL_Delay(delayTime);
}
视频线程会等待音频线程 比
较两者之间的pts 从而继续进行下一帧的播放
目前待解决问题：
1 AVI格式视频无法播放
参考文章：http://blog.yundiantech.com/?log=blog&scat=182
=======================================================================
ffmpeg视频H264压缩，rtsp推流课程教学视频：
https://edu.csdn.net/course/detail/27795
课件中里面提供源码可以直接下载运行！
=======================================================================
最近新开的公众号，文章正在一篇篇的更新，
公众号名称：玩转电子世界
各位朋友有什么问题了可以直接在上面提问，我会一一进行解答的。
跟着阳光非宅男，一步步走进电子的世界。
关注之后回复 资料下载 关键词可以获得免费海量的视频学习资料下载~~！
已共享的学习视频资料，共享资料正在不断更新中。
共享ffmpeg视频学习资料：
=======================================================================