上一篇让我们能运行起来kxmovie 项目.接下来我们就应该分析下如何播放音频和视频了.
文件概览
KxMovieViewController.h
KxMovieViewController.m
KxMovieGLView.h
KxMovieGLView.m
KxMovieDecoder.h
KxMovieDecoder.m
KxAudioManager.h
KxAudioManager.m
KxLogger.h
类概览
KxMovieViewController
KxMovieFrame
KxAudioFrame
KxVideoFrame
KxVideoFrameRGB
KxVideoFrameYUV
KxArtworkFrame
KxSubtitleFrame
KxMovieDecoder
KxMovieSubtitleASSParser
KxAudioManager
KxAudioManagerImpl
KxMovieGLView
KxMovieGLRenderer_RGB
KxMovieGLRenderer_YUV
类结构图
类分析
KxMovieViewController
属性和变量
该类相当于播放器,属性比较多,比较庞大.
入口方法
+(id) movieViewControllerWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters;
该类是通过该方法进行创建类的
static NSMutableDictionary * gHistory;
+ (void)initialize
{
if (!gHistory)
gHistory = [NSMutableDictionary dictionary];
}
该类在加载的时候,初始化了一个全局变量gHistory
+ (id) movieViewControllerWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters
{
id audioManager = [KxAudioManager audioManager];
[audioManager activateAudioSession];
return [[KxMovieViewController alloc] initWithContentPath: path parameters: parameters];
}
初始化的时候,我们初始化了类
KxAudioManager.该类是单利类.
接着调用方法- (id) initWithContentPath: (NSString *) path parameters: (NSDictionary *) parameters
- (id) initWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters
{
NSAssert(path.length > 0, @"empty path");
self = [super initWithNibName:nil bundle:nil];
if (self) {
_moviePosition = 0;
// self.wantsFullScreenLayout = YES;
_parameters = parameters;
__weak KxMovieViewController *weakSelf = self;
KxMovieDecoder *decoder = [[KxMovieDecoder alloc] init];
decoder.interruptCallback = ^BOOL(){
__strong KxMovieViewController *strongSelf = weakSelf;
return strongSelf ? [strongSelf interruptDecoder] : YES;
};
dispatch_async(dispatch_get_global_queue(0, 0), ^{
NSError *error = nil;
[decoder openFile:path error:&error];
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf) {
dispatch_sync(dispatch_get_main_queue(), ^{
[strongSelf setMovieDecoder:decoder withError:error];
});
}
});
}
return self;
}
1 .该类初始化了KxMovieDecoder(解码器).并且设置是否需要中断回调函数.
2.开启全局线程 解码器打开文件
3 调用- (void) setMovieDecoder: (KxMovieDecoder *) decoder withError: (NSError *) error
4 .返回 self
- (void) setMovieDecoder: (KxMovieDecoder *) decoder
withError: (NSError *) error
{
LoggerStream(2, @"setMovieDecoder");
if (!error && decoder) {
_decoder = decoder;
_dispatchQueue = dispatch_queue_create("KxMovie", DISPATCH_QUEUE_SERIAL);
_videoFrames = [NSMutableArray array];
_audioFrames = [NSMutableArray array];
if (_decoder.subtitleStreamsCount) {
_subtitles = [NSMutableArray array];
}
if (_decoder.isNetwork) {
_minBufferedDuration = NETWORK_MIN_BUFFERED_DURATION;
_maxBufferedDuration = NETWORK_MAX_BUFFERED_DURATION;
} else {
_minBufferedDuration = LOCAL_MIN_BUFFERED_DURATION;
_maxBufferedDuration = LOCAL_MAX_BUFFERED_DURATION;
}
if (!_decoder.validVideo)
_minBufferedDuration *= 10.0; // increase for audio
// allow to tweak some parameters at runtime
if (_parameters.count) {
id val;
val = [_parameters valueForKey: KxMovieParameterMinBufferedDuration];
if ([val isKindOfClass:[NSNumber class]])
_minBufferedDuration = [val floatValue];
val = [_parameters valueForKey: KxMovieParameterMaxBufferedDuration];
if ([val isKindOfClass:[NSNumber class]])
_maxBufferedDuration = [val floatValue];
val = [_parameters valueForKey: KxMovieParameterDisableDeinterlacing];
if ([val isKindOfClass:[NSNumber class]])
_decoder.disableDeinterlacing = [val boolValue];
if (_maxBufferedDuration < _minBufferedDuration)
_maxBufferedDuration = _minBufferedDuration * 2;
}
LoggerStream(2, @"buffered limit: %.1f - %.1f", _minBufferedDuration, _maxBufferedDuration);
if (self.isViewLoaded) {
[self setupPresentView];
_progressLabel.hidden = NO;
_progressSlider.hidden = NO;
_leftLabel.hidden = NO;
_infoButton.hidden = NO;
if (_activityIndicatorView.isAnimating) {
[_activityIndicatorView stopAnimating];
// if (self.view.window)
[self restorePlay];
}
}
} else {
if (self.isViewLoaded && self.view.window) {
[_activityIndicatorView stopAnimating];
if (!_interrupted)
[self handleDecoderMovieError: error];
}
}
}
该函数根据解码器解析数据的结构设置相关变量
在该函数里调用- (void) setupPresentView.该函数是设置播放视频的view的
而- (void) restorePlay函数是对视频进行播放
- (void) setupPresentView
{
CGRect bounds = self.view.bounds;
if (_decoder.validVideo) {
_glView = [[KxMovieGLView alloc] initWithFrame:bounds decoder:_decoder];
}
if (!_glView) {
LoggerVideo(0, @"fallback to use RGB video frame and UIKit");
[_decoder setupVideoFrameFormat:KxVideoFrameFormatRGB];
_imageView = [[UIImageView alloc] initWithFrame:bounds];
_imageView.backgroundColor = [UIColor blackColor];
}
UIView *frameView = [self frameView];
frameView.contentMode = UIViewContentModeScaleAspectFit;
frameView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleTopMargin | UIViewAutoresizingFlexibleRightMargin | UIViewAutoresizingFlexibleLeftMargin | UIViewAutoresizingFlexibleHeight | UIViewAutoresizingFlexibleBottomMargin;
[self.view insertSubview:frameView atIndex:0];
if (_decoder.validVideo) {
[self setupUserInteraction];
} else {
_imageView.image = [UIImage imageNamed:@"kxmovie.bundle/music_icon.png"];
_imageView.contentMode = UIViewContentModeCenter;
}
self.view.backgroundColor = [UIColor clearColor];
if (_decoder.duration == MAXFLOAT) {
_leftLabel.text = @"\u221E"; // infinity
_leftLabel.font = [UIFont systemFontOfSize:14];
CGRect frame;
frame = _leftLabel.frame;
frame.origin.x += 40;
frame.size.width -= 40;
_leftLabel.frame = frame;
frame =_progressSlider.frame;
frame.size.width += 40;
_progressSlider.frame = frame;
} else {
[_progressSlider addTarget:self
action:@selector(progressDidChange:)
forControlEvents:UIControlEventValueChanged];
}
if (_decoder.subtitleStreamsCount) {
CGSize size = self.view.bounds.size;
_subtitlesLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, size.height, size.width, 0)];
_subtitlesLabel.numberOfLines = 0;
_subtitlesLabel.backgroundColor = [UIColor clearColor];
_subtitlesLabel.opaque = NO;
_subtitlesLabel.adjustsFontSizeToFitWidth = NO;
_subtitlesLabel.textAlignment = NSTextAlignmentCenter;
_subtitlesLabel.autoresizingMask = UIViewAutoresizingFlexibleWidth;
_subtitlesLabel.textColor = [UIColor whiteColor];
_subtitlesLabel.font = [UIFont systemFontOfSize:16];
_subtitlesLabel.hidden = YES;
[self.view addSubview:_subtitlesLabel];
}
}
该函数就是设置播放视频的view和UI绘制
- (void) restorePlay
{
NSNumber *n = [gHistory valueForKey:_decoder.path];
if (n)
[self updatePosition:n.floatValue playMode:YES];
else
[self play];
}
播放视频
-(void) play
{
if (self.playing)
return;
if (!_decoder.validVideo &&
!_decoder.validAudio) {
return;
}
if (_interrupted)
return;
self.playing = YES;
_interrupted = NO;
_disableUpdateHUD = NO;
_tickCorrectionTime = 0;
_tickCounter = 0;
#ifdef DEBUG
_debugStartTime = -1;
#endif
[self asyncDecodeFrames];
[self updatePlayButton];
dispatch_time_t popTime = dispatch_time(DISPATCH_TIME_NOW, 0.1 * NSEC_PER_SEC);
dispatch_after(popTime, dispatch_get_main_queue(), ^(void){
[self tick];
});
if (_decoder.validAudio)
[self enableAudio:YES];
LoggerStream(1, @"play movie");
}
该函数 调用asyncDecodeFrames 解码视频
调用- (void) enableAudio: (BOOL) on 解码音频
- (void) asyncDecodeFrames
{
if (self.decoding)
return;
__weak KxMovieViewController *weakSelf = self;
__weak KxMovieDecoder *weakDecoder = _decoder;
const CGFloat duration = _decoder.isNetwork ? .0f : 0.1f;
self.decoding = YES;
dispatch_async(_dispatchQueue, ^{
{
__strong KxMovieViewController *strongSelf = weakSelf;
if (!strongSelf.playing)
return;
}
BOOL good = YES;
while (good) {
good = NO;
@autoreleasepool {
__strong KxMovieDecoder *decoder = weakDecoder;
if (decoder && (decoder.validVideo || decoder.validAudio)) {
NSArray *frames = [decoder decodeFrames:duration];
if (frames.count) {
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf)
good = [strongSelf addFrames:frames];
}
}
}
}
{
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf) strongSelf.decoding = NO;
}
});
}
在该函数中让解码器decoder对间隔时间进行解码.获取解码数组图像
调用 - (BOOL) addFrames: (NSArray *)frames.将解码的视频增加到缓存中
- (BOOL) addFrames: (NSArray *)frames
{
if (_decoder.validVideo) {
@synchronized(_videoFrames) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeVideo) {
[_videoFrames addObject:frame];
_bufferedDuration += frame.duration;
}
}
}
if (_decoder.validAudio) {
@synchronized(_audioFrames) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeAudio) {
[_audioFrames addObject:frame];
if (!_decoder.validVideo)
_bufferedDuration += frame.duration;
}
}
if (!_decoder.validVideo) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeArtwork)
self.artworkFrame = (KxArtworkFrame *)frame;
}
}
if (_decoder.validSubtitles) {
@synchronized(_subtitles) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeSubtitle) {
[_subtitles addObject:frame];
}
}
}
return self.playing && _bufferedDuration < _maxBufferedDuration;
}
将音视频对象加入到指定缓存中
- (void) enableAudio: (BOOL) on
{
id audioManager = [KxAudioManager audioManager];
if (on && _decoder.validAudio) {
audioManager.outputBlock = ^(float *outData, UInt32 numFrames, UInt32 numChannels) {
[self audioCallbackFillData: outData numFrames:numFrames numChannels:numChannels];
};
[audioManager play];
LoggerAudio(2, @"audio device smr: %d fmt: %d chn: %d",
(int)audioManager.samplingRate,
(int)audioManager.numBytesPerSample,
(int)audioManager.numOutputChannels);
} else {
[audioManager pause];
audioManager.outputBlock = nil;
}
}
播放音频
这就是完整的播放逻辑. UML时序图
时序图中只列举了相关关键函数的调用.许多变量的初始化都没详细书写
变量
_dispatchQueue 解码队列
_videoFrames 视频帧数组
_audioFrames 音频镇数组
解码器 KxMovieDecoder
属性和变量
KxMovieDecoder属性和变量
入口方法
+ (id) movieDecoderWithContentPath: (NSString *) path
error: (NSError **) perror;
- (BOOL) openFile: (NSString *) path
error: (NSError **) perror;
-init;
其实第一个方法就是下面两个方法的组合体
+ (id) movieDecoderWithContentPath: (NSString *) path
error: (NSError **) perror
{
KxMovieDecoder *mp = [[KxMovieDecoder alloc] init];
if (mp) {
[mp openFile:path error:perror];
}
return mp;
}
该类在加载 的时候初始化了ffmpeg
+ (void)initialize
{
av_log_set_callback(FFLog);
av_register_all();
avformat_network_init();
}
- av_log_set_callback 设置log回调函数
- av_register_all 注册ffmpeg组件
- avformat_network_init 默认开始网络模块
我们知道av_register_all 是在文件av_format.h文件中.我们看看这个类的介绍
av_format.h头文件有该类的基本使用
该库的主要作用是I/O 和 Muxing/Demuxing
该库有两个目的就是编解码音视频
使用该库我们首先必须要调用av_register_all()函数注册所有的编解码组件,如果我们需要网络模块,那么需要调用avformat_network_init()
AVInputFormat struct代表input format .AVOutputFormat struct代表 output format. 我们可以通过av_iformat_next()和av_oformat_next()查看我们注册的所有的input 和output format
我们可以通过avio_enum_protocols()查看该库支持的协议
AVFormatContext 是muxing 的和 demuxing 桥梁,用来读写文件的.
AVFormatContext 必须通过avformat_alloc_context()来创建.不能在栈上或者 用av_malloc()来创建.
AVFormatContext.iformat是对输入的引用
AVFormatContext.oformat是对输出的引用
input(iformat) 可以自动设置或者用户设置.但是output(oformat) 总是用户设置
AVFormatContext.streams是个数组,装有AVStream.该属性一般是通过下标来访问的
AVFormatContext.pb 是I/o 的上下文,可以打开该库和用户设置input和output.
url 字符串是代表scheme/协议. url需要有
:.如果没有:,那么认为是本地文件,但是这种写法被废弃了.想代表本地文件,需要使用file:格式
请注意，一些scheme/protocol非常强大，允许访问本地和远程文件、部分文件、本地音频和视频设备等。
解码 Demuxing
Demuxers 从音视频文件中读取数据并将其分成chunks 数据 ,该分割的数据用
AVPacket代表.AVPacket包含一帧或者多帧.
我们通过avformat_open_input()打开输入源.
av_read_frame()读取AVPacket
avformat_close_input()关闭数据源
代码片段打开媒体文件
const char *url = "file:in.mp3"; AVFormatContext *s = NULL; int ret = avformat_open_input(&s, url, NULL, NULL); if (ret < 0) abort();首先该函数先给AVFormatContext 分配内存,然后打开文件读取头文件,自动将其赋值到AVFormatContext 的iformat上.对于没有头文件的输入格式或者信息不够的,我们可以调用
avformat_find_stream_info函数,该函数能试着读取和解码一些frame,从中找到我们丢失的信息.
如果预先通过avformat_alloc_context函数创建了AVFormatContext.我们在传递给avformat_open_input之前需要做一些调整. 如果我们想用自定义函数来读取输入的数据.那么我们需要通过函数avio_alloc_context创建自己的AVIOContext,然后通过callbacks 函数来完成自定义函数对数据的解析.我们需要通过AVFormatContext.pb 来指向我们创建的新的AVIOContext
许多输入格式在avformat_open_input 返回之前我们是不知道具体的格式信息的.因此我们在已经预先创建的AVFormatContext不能设置demuxer private. 因此想要设置这些参数,我们需要通过
avformat_open_input函数将这些参数设定进去.AVDictionary *options = NULL; av_dict_set(&options, "video_size", "640x480", 0); av_dict_set(&options, "pixel_format", "rgb24", 0); if (avformat_open_input(&s, url, NULL, &options) < 0) abort(); av_dict_free(&options);上述代码我们给demuxer设置私有参数video_size 和pixel_format .
对于the rawvideo demuxer 是必须的,要不他不知道怎么解释视频源数据. 如果这些参数设置与原始数据的不同,那么这些参数也就不起作用了.不会识别的选项会在可选字典中返回(识别的会在字典中删除).
下面的代码片段是处理不认识的可选参数AVDictionaryEntry *e; if (e = av_dict_get(options, "", NULL, AV_DICT_IGNORE_SUFFIX)) { fprintf(stderr, "Option %s not recognized by the demuxer.\n", e->key); abort(); }
如何读取数据
我们可以反复调用av_read_frame函数获取数据.调用该函数,要是成功就返回AVPacket数据.我们可以通过调用avcodec_send_packet或者avcodec_decode_subtitle2来解码AVPacket
AVPacket.pts, AVPacket.dts 和 AVPacket.duration 的时间信息会在解码的时候设置.(解码的时候知道的前提).如果流中没有提供这些信息,这些值可能被重新设置.这些时间信息的基本单位是AVStream.time_base(都是该值的整数倍).需要和该值相乘换算成秒
要是AVPacket.buf的在返回的packet中被设置.那么该空间是动态分配的,用户可以无限期的持有它.如果AVPacket.buf是NULL,那么返回的是一个在demuxer中的静态区域.该AVPacket有效期到下一次调用av_read_frame.如果调用者需要一个长的时间周期保持AVPacket.那么我们可以通过调用av_dup_packet 来复制一份出来
我们需要调用av_packet_unref来释放该空间.
解码 Muxing
Muxers 可以解码AVPacket 数据将其写入文件或者指定的输出格式数据
avformat_write_header()写文件的头文件
av_write_frame() / av_interleaved_write_frame()写packets数据到文件
av_write_trailer()写文件结尾
开启解码Muxing ,首先要调用avformat_alloc_context创建上下文.
我们需要给AVFormatContext.oformat附上该值
并且我们也要设置AVFormatContext.pb 的值,该值必须是avio_open2()返回的上下文或者自定义的上下文
我们需要通过avformat_new_stream()创建一个stream.我们应该给创建好的steam的属性codecpar 赋值(该参数是流编码需要的参数).
AVStream.time_base 也需要设置.
大概流程如下
这里是真正的初始化函数
- (BOOL) openFile: (NSString *) path
error: (NSError **) perror
{
NSAssert(path, @"nil path");
NSAssert(!_formatCtx, @"already open");
_isNetwork = isNetworkPath(path);
static BOOL needNetworkInit = YES;
if (needNetworkInit && _isNetwork) {
needNetworkInit = NO;
avformat_network_init();
}
_path = path;
kxMovieError errCode = [self openInput: path];
if (errCode == kxMovieErrorNone) {
kxMovieError videoErr = [self openVideoStream];
kxMovieError audioErr = [self openAudioStream];
_subtitleStream = -1;
if (videoErr != kxMovieErrorNone &&
audioErr != kxMovieErrorNone) {
errCode = videoErr; // both fails
} else {
_subtitleStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_SUBTITLE);
}
}
if (errCode != kxMovieErrorNone) {
[self closeFile];
NSString *errMsg = errorMessage(errCode);
LoggerStream(0, @"%@, %@", errMsg, path.lastPathComponent);
if (perror)
*perror = kxmovieError(errCode, errMsg);
return NO;
}
return YES;
}
分步解释
1 isNetworkPath 是判断是否是需要网络模块. 需要网络模块加载
avformat_network_init
- 调用函数
- (kxMovieError) openInput: (NSString *) path打开文件
3 打开openVideoStream
4 打开openAudioStream
5 _subtitleStreams 参数设置
流程很清晰,分别看
- (kxMovieError) openInput: (NSString *) path
{
AVFormatContext *formatCtx = NULL;
if (_interruptCallback) {
formatCtx = avformat_alloc_context();
if (!formatCtx)
return kxMovieErrorOpenFile;
AVIOInterruptCB cb = {interrupt_callback, (__bridge void *)(self)};
formatCtx->interrupt_callback = cb;
}
if (avformat_open_input(&formatCtx, [path cStringUsingEncoding: NSUTF8StringEncoding], NULL, NULL) < 0) {
if (formatCtx)
avformat_free_context(formatCtx);
return kxMovieErrorOpenFile;
}
if (avformat_find_stream_info(formatCtx, NULL) < 0) {
avformat_close_input(&formatCtx);
return kxMovieErrorStreamInfoNotFound;
}
av_dump_format(formatCtx, 0, [path.lastPathComponent cStringUsingEncoding: NSUTF8StringEncoding], false);
_formatCtx = formatCtx;
return kxMovieErrorNone;
}
这里我们在外界设置了_interruptCallback 参数,因此,这里我们采用的是用户创建AVFormatContext 结构体
1.创建AVFormatContext 对象,并且设置interrupt_callback回调函数
2.调用avformat_open_input打开文件
3.调用函数avformat_find_stream_info检查对象AVFormatContext 是否有对应的解码器
4.调用av_dump_format 打印信息
- (kxMovieError) openVideoStream
{
kxMovieError errCode = kxMovieErrorStreamNotFound;
_videoStream = -1;
_artworkStream = -1;
_videoStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_VIDEO);
for (NSNumber *n in _videoStreams) {
const NSUInteger iStream = n.integerValue;
if (0 == (_formatCtx->streams[iStream]->disposition & AV_DISPOSITION_ATTACHED_PIC)) {
errCode = [self openVideoStream: iStream];
if (errCode == kxMovieErrorNone)
break;
} else {
_artworkStream = iStream;
}
}
return errCode;
}
1 从_formatCtx->stream中查找包含AVMEDIA_TYPE_VIDEO的AVStream
2.依次判断AVStream的disposition 是否支持 AV_DISPOSITION_ATTACHED_PIC(图像)
3 支持图像调用- (kxMovieError) openVideoStream: (NSInteger) videoStream
- (kxMovieError) openVideoStream: (NSInteger) videoStream
{
// get a pointer to the codec context for the video stream
AVCodecContext *codecCtx = _formatCtx->streams[videoStream]->codec;
// find the decoder for the video stream
AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
if (!codec)
return kxMovieErrorCodecNotFound;
// inform the codec that we can handle truncated bitstreams -- i.e.,
// bitstreams where frame boundaries can fall in the middle of packets
//if(codec->capabilities & CODEC_CAP_TRUNCATED)
// _codecCtx->flags |= CODEC_FLAG_TRUNCATED;
// open codec
if (avcodec_open2(codecCtx, codec, NULL) < 0)
return kxMovieErrorOpenCodec;
_videoFrame = av_frame_alloc();
if (!_videoFrame) {
avcodec_close(codecCtx);
return kxMovieErrorAllocateFrame;
}
_videoStream = videoStream;
_videoCodecCtx = codecCtx;
// determine fps
AVStream *st = _formatCtx->streams[_videoStream];
avStreamFPSTimeBase(st, 0.04, &_fps, &_videoTimeBase);
LoggerVideo(1, @"video codec size: %d:%d fps: %.3f tb: %f",
self.frameWidth,
self.frameHeight,
_fps,
_videoTimeBase);
LoggerVideo(1, @"video start time %f", st->start_time * _videoTimeBase);
LoggerVideo(1, @"video disposition %d", st->disposition);
return kxMovieErrorNone;
}
1获取AVCodecContext 对象
2 调用avcodec_find_decoder查找解码器
3.avcodec_open2打开解码器
- 调用avStreamFPSTimeBase 设基准时间
- (kxMovieError) openAudioStream
{
kxMovieError errCode = kxMovieErrorStreamNotFound;
_audioStream = -1;
_audioStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_AUDIO);
for (NSNumber *n in _audioStreams) {
errCode = [self openAudioStream: n.integerValue];
if (errCode == kxMovieErrorNone)
break;
}
return errCode;
}
音频也是一样的查找过程只不过查找的格式是AVMEDIA_TYPE_AUDIO
- (kxMovieError) openAudioStream: (NSInteger) audioStream
{
AVCodecContext *codecCtx = _formatCtx->streams[audioStream]->codec;
SwrContext *swrContext = NULL;
AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
if(!codec)
return kxMovieErrorCodecNotFound;
if (avcodec_open2(codecCtx, codec, NULL) < 0)
return kxMovieErrorOpenCodec;
if (!audioCodecIsSupported(codecCtx)) {
id audioManager = [KxAudioManager audioManager];
swrContext = swr_alloc_set_opts(NULL,
av_get_default_channel_layout(audioManager.numOutputChannels),
AV_SAMPLE_FMT_S16,
audioManager.samplingRate,
av_get_default_channel_layout(codecCtx->channels),
codecCtx->sample_fmt,
codecCtx->sample_rate,
0,
NULL);
if (!swrContext ||
swr_init(swrContext)) {
if (swrContext)
swr_free(&swrContext);
avcodec_close(codecCtx);
return kxMovieErroReSampler;
}
}
_audioFrame = av_frame_alloc();
if (!_audioFrame) {
if (swrContext)
swr_free(&swrContext);
avcodec_close(codecCtx);
return kxMovieErrorAllocateFrame;
}
_audioStream = audioStream;
_audioCodecCtx = codecCtx;
_swrContext = swrContext;
AVStream *st = _formatCtx->streams[_audioStream];
avStreamFPSTimeBase(st, 0.025, 0, &_audioTimeBase);
LoggerAudio(1, @"audio codec smr: %.d fmt: %d chn: %d tb: %f %@",
_audioCodecCtx->sample_rate,
_audioCodecCtx->sample_fmt,
_audioCodecCtx->channels,
_audioTimeBase,
_swrContext ? @"resample" : @"");
return kxMovieErrorNone;
}
1.调用
avcodec_find_decoder查找音频解码器
2.调用avcodec_open2 打开音频解码器
- 设置音频到KxAudioManager对象
- 设置片段基本时间
流程框图如下
从上面的时序图我们能看看出来avformat 解码的基本流程
注册所有组件 -> 打开网络组件-> 打开文件-> 查找解码器->解码
这里我们需要看下函数static void avStreamFPSTimeBase(AVStream *st, CGFloat defaultTimeBase, CGFloat *pFPS, CGFloat *pTimeBase)设置解码片段时间
static void avStreamFPSTimeBase(AVStream *st, CGFloat defaultTimeBase, CGFloat *pFPS, CGFloat *pTimeBase)
{
CGFloat fps, timebase;
if (st->time_base.den && st->time_base.num)
timebase = av_q2d(st->time_base);
else if(st->codec->time_base.den && st->codec->time_base.num)
timebase = av_q2d(st->codec->time_base);
else
timebase = defaultTimeBase;
if (st->codec->ticks_per_frame != 1) {
LoggerStream(0, @"WARNING: st.codec.ticks_per_frame=%d", st->codec->ticks_per_frame);
//timebase *= st->codec->ticks_per_frame;
}
if (st->avg_frame_rate.den && st->avg_frame_rate.num)
fps = av_q2d(st->avg_frame_rate);
else if (st->r_frame_rate.den && st->r_frame_rate.num)
fps = av_q2d(st->r_frame_rate);
else
fps = 1.0 / timebase;
if (pFPS)
*pFPS = fps;
if (pTimeBase)
*pTimeBase = timebase;
}
AVStream.time_base 解码过程中是被avformat设置的代表时间戳
AVStream. avg_frame_rate 设置解码的fps
以上没有真正的解码过程.这里我们看看视频的真正解码
- (NSArray *) decodeFrames: (CGFloat) minDuration
{
if (_videoStream == -1 &&
_audioStream == -1)
return nil;
NSMutableArray *result = [NSMutableArray array];
AVPacket packet;
CGFloat decodedDuration = 0;
BOOL finished = NO;
while (!finished) {
if (av_read_frame(_formatCtx, &packet) < 0) {
_isEOF = YES;
break;
}
if (packet.stream_index ==_videoStream) {
int pktSize = packet.size;
while (pktSize > 0) {
int gotframe = 0;
int len = avcodec_decode_video2(_videoCodecCtx,
_videoFrame,
&gotframe,
&packet);
if (len < 0) {
LoggerVideo(0, @"decode video error, skip packet");
break;
}
if (gotframe) {
if (!_disableDeinterlacing &&
_videoFrame->interlaced_frame) {
// avpicture_deinterlace((AVPicture*)_videoFrame,
// (AVPicture*)_videoFrame,
// _videoCodecCtx->pix_fmt,
// _videoCodecCtx->width,
// _videoCodecCtx->height);
}
KxVideoFrame *frame = [self handleVideoFrame];
if (frame) {
[result addObject:frame];
_position = frame.position;
decodedDuration += frame.duration;
if (decodedDuration > minDuration)
finished = YES;
}
}
if (0 == len)
break;
pktSize -= len;
}
} else if (packet.stream_index == _audioStream) {
int pktSize = packet.size;
while (pktSize > 0) {
int gotframe = 0;
int len = avcodec_decode_audio4(_audioCodecCtx,
_audioFrame,
&gotframe,
&packet);
if (len < 0) {
LoggerAudio(0, @"decode audio error, skip packet");
break;
}
if (gotframe) {
KxAudioFrame * frame = [self handleAudioFrame];
if (frame) {
[result addObject:frame];
if (_videoStream == -1) {
_position = frame.position;
decodedDuration += frame.duration;
if (decodedDuration > minDuration)
finished = YES;
}
}
}
if (0 == len)
break;
pktSize -= len;
}
} else if (packet.stream_index == _artworkStream) {
if (packet.size) {
KxArtworkFrame *frame = [[KxArtworkFrame alloc] init];
frame.picture = [NSData dataWithBytes:packet.data length:packet.size];
[result addObject:frame];
}
} else if (packet.stream_index == _subtitleStream) {
int pktSize = packet.size;
while (pktSize > 0) {
AVSubtitle subtitle;
int gotsubtitle = 0;
int len = avcodec_decode_subtitle2(_subtitleCodecCtx,
&subtitle,
&gotsubtitle,
&packet);
if (len < 0) {
LoggerStream(0, @"decode subtitle error, skip packet");
break;
}
if (gotsubtitle) {
KxSubtitleFrame *frame = [self handleSubtitle: &subtitle];
if (frame) {
[result addObject:frame];
}
avsubtitle_free(&subtitle);
}
if (0 == len)
break;
pktSize -= len;
}
}
av_free_packet(&packet);
}
return result;
}
这个函数的调用是在KxMovieViewController类的专有_dispatchQueue中进行解码的.并且执行该函数,我们已经找到了音视频的专有的解码器了.这里讲解我们如何获取音视频的AVPacket
调用
av_read_frame获取AVPacket
判断AVPacket是否是我们需要解码的包,不是就继续读取包
对于视频调用avcodec_decode_video2对视频解码.数据帧存放在_videoFrame变量 中. 调用handleVideoFrame获取KxVideoFrame对象存入到数组中avcodec_decode_video2 在最新的4.3 版本被废弃了.可以使用avcodec_send_packet() and avcodec_receive_frame()来解码
音频和视频都是同样的道理
这里我们看如何处理vidoeFrame的
- (KxVideoFrame *) handleVideoFrame
{
if (!_videoFrame->data[0])
return nil;
KxVideoFrame *frame;
if (_videoFrameFormat == KxVideoFrameFormatYUV) {
KxVideoFrameYUV * yuvFrame = [[KxVideoFrameYUV alloc] init];
yuvFrame.luma = copyFrameData(_videoFrame->data[0],
_videoFrame->linesize[0],
_videoCodecCtx->width,
_videoCodecCtx->height);
yuvFrame.chromaB = copyFrameData(_videoFrame->data[1],
_videoFrame->linesize[1],
_videoCodecCtx->width / 2,
_videoCodecCtx->height / 2);
yuvFrame.chromaR = copyFrameData(_videoFrame->data[2],
_videoFrame->linesize[2],
_videoCodecCtx->width / 2,
_videoCodecCtx->height / 2);
frame = yuvFrame;
} else {
if (!_swsContext &&
![self setupScaler]) {
LoggerVideo(0, @"fail setup video scaler");
return nil;
}
sws_scale(_swsContext,
(const uint8_t **)_videoFrame->data,
_videoFrame->linesize,
0,
_videoCodecCtx->height,
_picture.data,
_picture.linesize);
KxVideoFrameRGB *rgbFrame = [[KxVideoFrameRGB alloc] init];
rgbFrame.linesize = _picture.linesize[0];
rgbFrame.rgb = [NSData dataWithBytes:_picture.data[0]
length:rgbFrame.linesize * _videoCodecCtx->height];
frame = rgbFrame;
}
frame.width = _videoCodecCtx->width;
frame.height = _videoCodecCtx->height;
frame.position = av_frame_get_best_effort_timestamp(_videoFrame) * _videoTimeBase;
const int64_t frameDuration = av_frame_get_pkt_duration(_videoFrame);
if (frameDuration) {
frame.duration = frameDuration * _videoTimeBase;
frame.duration += _videoFrame->repeat_pict * _videoTimeBase * 0.5;
//if (_videoFrame->repeat_pict > 0) {
// LoggerVideo(0, @"_videoFrame.repeat_pict %d", _videoFrame->repeat_pict);
//}
} else {
// sometimes, ffmpeg unable to determine a frame duration
// as example yuvj420p stream from web camera
frame.duration = 1.0 / _fps;
}
#if 0
LoggerVideo(2, @"VFD: %.4f %.4f | %lld ",
frame.position,
frame.duration,
av_frame_get_pkt_pos(_videoFrame));
#endif
return frame;
}
这里我们采用的是_videoFrameFormat 变量的值是KxVideoFrameFormatYUV.在KxMovieGLView 中设置的
KxVideoFrameFormatYUV对应的avformat中的AV_PIX_FMT_YUV420P
AV_PIX_FMT_YUV420P的排列格式是< planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
我们对_videoFrame中的数据解析到到KxVideoFrameYUV 中
这里我们需要了解下yuv420 的视频流格式.可以参考这里
bpp 位图的单位
16 bit per pixel
每个像素点分解为16个字节来表示
接下来看音频编码
- (KxAudioFrame *) handleAudioFrame
{
if (!_audioFrame->data[0])
return nil;
id audioManager = [KxAudioManager audioManager];
const NSUInteger numChannels = audioManager.numOutputChannels;
NSInteger numFrames;
void * audioData;
if (_swrContext) {
const NSUInteger ratio = MAX(1, audioManager.samplingRate / _audioCodecCtx->sample_rate) *
MAX(1, audioManager.numOutputChannels / _audioCodecCtx->channels) * 2;
const int bufSize = av_samples_get_buffer_size(NULL,
audioManager.numOutputChannels,
_audioFrame->nb_samples * ratio,
AV_SAMPLE_FMT_S16,
1);
if (!_swrBuffer || _swrBufferSize < bufSize) {
_swrBufferSize = bufSize;
_swrBuffer = realloc(_swrBuffer, _swrBufferSize);
}
Byte *outbuf[2] = { _swrBuffer, 0 };
numFrames = swr_convert(_swrContext,
outbuf,
_audioFrame->nb_samples * ratio,
(const uint8_t **)_audioFrame->data,
_audioFrame->nb_samples);
if (numFrames < 0) {
LoggerAudio(0, @"fail resample audio");
return nil;
}
//int64_t delay = swr_get_delay(_swrContext, audioManager.samplingRate);
//if (delay > 0)
// LoggerAudio(0, @"resample delay %lld", delay);
audioData = _swrBuffer;
} else {
if (_audioCodecCtx->sample_fmt != AV_SAMPLE_FMT_S16) {
NSAssert(false, @"bucheck, audio format is invalid");
return nil;
}
audioData = _audioFrame->data[0];
numFrames = _audioFrame->nb_samples;
}
const NSUInteger numElements = numFrames * numChannels;
NSMutableData *data = [NSMutableData dataWithLength:numElements * sizeof(float)];
float scale = 1.0 / (float)INT16_MAX ;
vDSP_vflt16((SInt16 *)audioData, 1, data.mutableBytes, 1, numElements);
vDSP_vsmul(data.mutableBytes, 1, &scale, data.mutableBytes, 1, numElements);
KxAudioFrame *frame = [[KxAudioFrame alloc] init];
frame.position = av_frame_get_best_effort_timestamp(_audioFrame) * _audioTimeBase;
frame.duration = av_frame_get_pkt_duration(_audioFrame) * _audioTimeBase;
frame.samples = data;
if (frame.duration == 0) {
// sometimes ffmpeg can't determine the duration of audio frame
// especially of wma/wmv format
// so in this case must compute duration
frame.duration = frame.samples.length / (sizeof(float) * numChannels * audioManager.samplingRate);
}
#if 0
LoggerAudio(2, @"AFD: %.4f %.4f | %.4f ",
frame.position,
frame.duration,
frame.samples.length / (8.0 * 44100.0));
#endif
return frame;
}
- (NSArray *) decodeFrames: (CGFloat) minDuration 函数就是对AVPacket 数据进行解码,保存到各自的音频和视频解码数组中
KxMovieGLView
ffmeng支持协议列表