tutorial 5 音视频同步 初印象
同步视频基础
时间戳
dts解码时间戳
pts显示时间戳
动手实践目标
-
想不想动手实践看看pts第一次 被赋值。
-
重点研究那个video_fresh-timer函数
-
鉴于分析的视频只有12帧，
-
理解actual_delay 期望值
-
Audio的时间
-
理解一下那个 时间戳 vs音频时间的 言论
-
-
video_fresh-timer关注delay :（现在的 时间戳-上一个时间戳）同步阈值 两个变量
-
提前告知 tutorial同步音频到视频，
-
认识校正方法，是修正delay,要么加倍延迟，要么没有延迟。下一节对于音频来说，要么插值，要么删减？一句话，同步的手段各有。
表面变化：代码变化（tutorial 4 vs tutorial 5）
增加的宏如下：
#define AV_SYNC_THRESHOLD 0.010 //minimum sync time #define AV_NOSYNC_THRESHOLD 10 //the differ of video and audio exceeds the time, do not sync with audio
值得注意的数据结构变化如下：
struct VideoState{
...
double video_clock;
double audio_clock;
double frame_last_pts;
double frame_last_delay;
double frame_timer;
} VideoState;
关于音频当前时钟字段，暂时忽略，详见官方tutorial
视频线程代码及变迁
int video_thread(void *arg) {
VideoState *is = (VideoState *) arg;
AVPacket pkt1, *packet = &pkt1;
int len1, frameFinished;
AVFrame *pFrame;
double pts = 0;
pFrame = avcodec_alloc_frame();
for (;;) {
if (packet_queue_get(&is->videoq, packet, 1) < 0) {
// means we quit getting packets
break;
}
pts = 0;
//save global pts to be stored in pFrame in first call
global_video_pkt_pts = packet->pts;
// printf("packet->pts %d\n", packet->pts );
// printf("packet->dts %d\n", packet->dts );
// Decode video frame
len1 = avcodec_decode_video2(is->video_st->codec, pFrame,
&frameFinished, packet);
if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque
&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE) {
pts = *(uint64_t*) pFrame->opaque;
} else if (packet->dts != AV_NOPTS_VALUE) {
pts = packet->dts;
} else {
pts = 0;
}
// printf("pts %d\n", pts );
pts *= av_q2d(is->video_st->time_base);
// Did we get a video frame?
if (frameFinished) {
pts = synchronize_video(is, pFrame, pts);
if (queue_picture(is, pFrame, pts) < 0) {
break;
}
}
av_free_packet(packet);
}
av_free(pFrame);
return 0;
}
差别有如下：
Pts值的初始化,详细如下图
图中红色部分是额外添加的dts处理
下图中表明还有视频同步函数的加入。
以上是所有的两处大的修改。
视频刷新定时器video_refresh_timer
一句话：请看英文解释，一次性解释清楚
| void video_refresh_timer(void *userdata) {
VideoState *is = (VideoState *) userdata; VideoPicture *vp; double actual_delay, delay, sync_threshold, ref_clock, diff;
if (is->video_st) { if (is->pictq_size == 0) { schedule_refresh(is, 1); } else { vp = &is->pictq[is->pictq_rindex];
/* Timing code goes here */ delay = vp->pts - is->frame_last_pts; //the pts from last if (delay <= 0 || delay >= 1.0) { //if incorrect delay, use previous one过大或者多小，超过1s,不过理喻 delay = is->frame_last_delay; } /* save for next time */ is->frame_last_delay = delay; is->frame_last_pts = vp->pts;
/* update delay to sync to audio */ ref_clock = get_audio_clock(is); diff = vp->pts - ref_clock;
/* Skip or repeat the frame. Take delay into account FFPlay still doesn't "know if this is the best guess." */ sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay : AV_SYNC_THRESHOLD; if (fabs(diff) < AV_NOSYNC_THRESHOLD) { if (diff <= -sync_threshold) { delay = 0; //video is slow, do not delay，视频已太慢（视频预测时间-当前时间 都是负值，请赶紧播） } else if (diff >= sync_threshold) delay = 2 * delay; //video is fast, delay double，视频太快，刷新时间变慢点 } } is->frame_timer += delay; /* computer the REAL delay */ actual_delay = is->frame_timer - (av_gettime() / 1000000.0); if (actual_delay < 0.010) { /* Really it should skip the picture instead */ actual_delay = 0.010; } schedule_refresh(is, (int) (actual_delay * 1000 + 0.5)); // schedule_refresh(is, 80);
/* show the picture! */ video_display(is);
/* update queue for next picture! */ 后续略 }
|
实验结果分析，
Video_thread代码：
所有亮点都在颜色里。
Video_thread{
...
if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque
&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE) {
pts = *(uint64_t*) pFrame->opaque;
} else if (packet->dts != AV_NOPTS_VALUE) {
pts = packet->dts;//12帧测试案列执行此代码覆盖
} else {
pts = 0;
}
pts *= av_q2d(is->video_st->time_base);
// Did we get a video frame?
if (frameFinished) {
pts = synchronize_video(is, pFrame, pts);
//除第一帧没有pts赋值，其他帧都有赋值。其他备注本案例没有重复帧，
if (queue_picture(is, pFrame, pts) < 0) {
break;
}
}
}
video_refresh_timer代码运行结果
这里的数据结果较多，尤其替换了上一小节定时刷新，值得反复体会。
void video_refresh_timer(void *userdata) {
...
if (is->video_st) {
if (is->pictq_size == 0) {
} else {
vp = &is->pictq[is->pictq_rindex];
/* Timing code goes here */
delay = vp->pts - is->frame_last_pts; //the pts from last
//实验结果非常理想，所有12帧处理的结果都是1.0
if (delay <= 0 || delay >= 1.0) {
//if incorrect delay, use previous one
delay= is->frame_last_delay;
}
/* save for next time */
is->frame_last_delay = delay;
is->frame_last_pts = vp->pts;
/* update delay to sync to audio */
ref_clock = get_audio_clock(is);
diff = vp->pts - ref_clock;//具体的值见下面的表格
/* Skip or repeat the frame. Take delay into account
FFPlay still doesn't "know if this is the best guess." */
sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay
: AV_SYNC_THRESHOLD;
if (fabs(diff) < AV_NOSYNC_THRESHOLD) {
if (diff <= -sync_threshold) {
delay = 0; //video is slow, do not delay //实验结果非常理想，都是说明视频太慢，
} else if (diff >= sync_threshold) {
delay = 2 * delay; //video is fast, delay double
}
}
is->frame_timer += delay;//鉴于12帧delay=0，这个值就没有增长过
/* computer the REAL delay */
actual_delay = is->frame_timer - (av_gettime() / 1000000.0);
if (actual_delay < 0.010) {//详细的真实delay数据如下，可见都是大大小于阈值
/* Really it should skip the picture instead */
actual_delay = 0.010;
}
logtime=actual_delay * 1000 + 0.5;
schedule_refresh(is, (int) (actual_delay * 1000 + 0.5));//所有填入的时间都是10.5
// schedule_refresh(is, 80);
/* show the picture! */
video_display(is);
//ingore the the other code
}
} else {
schedule_refresh(is, 100);
}
}
12帧的时延分析
表格，对于所有12帧的delay初步结果分析，可以看到所有的帧都太慢，不执行delay. 但是视觉上看来，视频帧还是太慢，这说明音频还得同视频同步，期待下一小节。
| // 10 //the differ of video and audio exceeds the time, do not sync with audio // if(diff <= -sync_threshold)所有出现的sync_threshold都是0.04 diff=-2.0393626991565137, //video is slow, do not delay， diff=-5.8978444236176193, //video is slow, do not delay， diff=-9.8762886597938149, //video is slow, do not delay， diff=-8.9662605435801321, //video is slow, do not delay， diff=-7.9662605435801321, //video is slow, do not delay， diff=-6.9662605435801321, //video is slow, do not delay， diff=-5.9662605435801321, //video is slow, do not delay， diff=-4.9662605435801321, //video is slow, do not delay， diff=-3.9662605435801321, //video is slow, do not delay， diff=-2.9662605435801321, //video is slow, do not delay， diff=-1.9662605435801321, //video is slow, do not delay， diff=-0.96626054358013214, //video is slow, do not delay |
在计算真实的delay时，涉及到的表格如下，
| actual_delay= -3.2161841392517090 ,frame_timer= 1401626707.2479169 actual_delay= -8.3304760456085205 ,frame_timer= 1401626707.2479169 actual_delay= -13.639780044555664 ,frame_timer= 1401626707.2479169 actual_delay= -14.218813180923462 ,frame_timer= 1401626707.2479169 actual_delay= -14.572834014892578 ,frame_timer= 1401626707.2479169 actual_delay= -14.932854175567627 ,frame_timer= 1401626707.2479169 actual_delay= -15.305875062942505 ,frame_timer= 1401626707.2479169 actual_delay= -15.662896156311035 ,frame_timer= 1401626707.2479169 actual_delay= -16.020915985107422 ,frame_timer= 1401626707.2479169 actual_delay= -16.381937026977539 ,frame_timer= 1401626707.2479169 actual_delay= -16.737957000732422 ,frame_timer= 1401626707.2479169 actual_delay= -16.947968959808350 ,frame_timer= 1401626707.2479169
|
其他：
当播放结束，没有额外的多调用刷屏函数，不像上一个版本。
结果中，也有多线程的随机性出现，情况在预料之中。
未完成分析日志log的一轮起始，下次补充就是。
现在认识 依然 很浅，比如i b b p的认识
小结：
逻辑很复杂，测试用例很理想，入门在路上。
不要忘了英文注释
不要忘了。。
视频即使同步了，也比音频慢！！
回顾，主要是看vedio_thread ,和播放的timer
附件完整log分析
简化版（没有带修正的log）
Function: video_thread(void *), [pts 2]=0
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=0.00000000000000000
Function: video_refresh_timer(void *), pts之delay=0.00000000000000000
Function: video_refresh_timer(void *), diff=-2.0393626991565137, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=1
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=1.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-5.8978444236176193, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=2
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=2.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-9.8762886597938149, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=3
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=3.0000000000000000
Function: video_thread(void *), [pts 2]=4
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=4.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-8.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=5
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=5.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-7.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=6
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=6.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-6.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=7
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=7.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-5.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=8
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=8.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-4.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=9
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=9.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-3.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=10
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=10.000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-2.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=11
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=11.000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-1.9662605435801321, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-0.96626054358013214, //video is slow, do not delay
Function: video_refresh_timer(void *),【core time】=10.500000000000000
最终用到的log
Function: our_get_buffer(AVCodecContext *, AVFrame *),pts初始化in自定义alloc，pts=0
Function: video_thread(void *), [pts 2]=0
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we aren't given a pts, set it to the clock */, 0.00000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=0.00000000000000000
Function: video_refresh_timer(void *), pts之delay=0.00000000000000000
Function: video_refresh_timer(void *), diff=-2.0393626991565137, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -3.2161841392517090 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=1
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 1.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=1.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-5.8978444236176193, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -8.3304760456085205 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=2
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 2.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=2.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-9.8762886597938149, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -13.639780044555664 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=3
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 3.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=3.0000000000000000
Function: video_thread(void *), [pts 2]=4
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 4.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=4.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-8.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.218813180923462 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=5
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 5.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=5.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-7.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.572834014892578 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=6
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 6.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=6.0000000000000000
Function: video_refresh_timer(void *), diff=-6.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.932854175567627 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=7
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 7.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=7.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-5.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -15.305875062942505 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=8
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 8.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=8.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-4.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -15.662896156311035 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=9
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 9.0000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=9.0000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-3.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.020915985107422 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=10
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 10.000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=10.000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-2.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.381937026977539 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_thread(void *), [pts 2]=11
Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 11.000000000000000
Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=11.000000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-1.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.737957000732422 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000
Function: video_refresh_timer(void *), pts之delay=1.0000000000000000
Function: video_refresh_timer(void *), diff=-0.96626054358013214, //video is slow, do not delay，0.040000000000000001 must>=10ms
Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.947968959808350 ,frame_timer= 1401626707.2479169
Function: video_refresh_timer(void *),【core time】=10.500000000000000