本文记录基于Video Toolbox的H264视频流硬件解码获取RGB888像素数据的方法。
h264
初始化Video Toolbox
初始化时重点在于attrs中的参数,attrs决定了回调返回的数据，将values中的v设为kCVPixelFormatType_32BGRA可以在回调中得到32bit BGRA数据，另外几个常用的枚举类型有为kCVPixelFormatType_420YpCbCr8Planar和kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,分别对于返回数据为YUV420和NV12。开发者尝试使用kCVPixelFormatType_32RGBA作为参数，测试发现回调中得不到CVPixelBufferRef,遂改用kCVPixelFormatType_32BGRA，最终达到预期。
完整参考代码如下：
-(BOOL)initH264Decoder
{
if(_deocderSession) {
return YES;
}
const uint8_t* const parameterSetPointers[2] = { _sps, _pps };
const size_t parameterSetSizes[2] = { _spsSize, _ppsSize };
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,
2, //param count
parameterSetPointers,
parameterSetSizes,
4, //nal start code size
&_decoderFormatDescription);
if(status == noErr) {
CFDictionaryRef attrs = NULL;
const void *keys[] = { kCVPixelBufferPixelFormatTypeKey };
// kCVPixelFormatType_420YpCbCr8Planar is YUV420
// kCVPixelFormatType_420YpCbCr8BiPlanarFullRange is NV12
uint32_t v = kCVPixelFormatType_32BGRA;
const void *values[] = { CFNumberCreate(NULL, kCFNumberSInt32Type, &v) };
attrs = CFDictionaryCreate(NULL, keys, values, 1, NULL, NULL);
VTDecompressionOutputCallbackRecord callBackRecord;
callBackRecord.decompressionOutputCallback = didDecompress;
callBackRecord.decompressionOutputRefCon = NULL;
status = VTDecompressionSessionCreate(kCFAllocatorDefault,
_decoderFormatDescription,
NULL, attire
&callBackRecord,
&_deocderSession);
CFRelease(attires
DDLogDebug(@"Current status:%d", (int)status);
} else {
DDLogDebug(@"IOS8VT: reset decoder session failed status=%d", (int)status);
}
return YES;
}
解码与处理回调
在子线程中处理解码逻辑：
dispatch_async(dispatch_get_global_queue(0, 0), ^{
[self decodeFile:fileName fileExt:@"h264"];
});
decodeFile:fileExt:方法体：
-(void)decodeFile:(NSString*)fileName fileExt:(NSString*)fileExt
{
NSInteger nalIndex = 0;
CFAbsoluteTime startTime =CFAbsoluteTimeGetCurrent();
NSString *path = [[NSBundle mainBundle] pathForResource:fileName ofType:fileExt];
MEVideoFileParser *parser = [MEVideoFileParser alloc];
[parser open:path];
DDLogDebug(@"Decode start");
VideoPacket *vp = nil;
while(true) {
vp = [parser nextPacket];
if(vp == nil) {
break;
}
uint32_t nalSize = (uint32_t)(vp.size - 4);
uint8_t *pNalSize = (uint8_t*)(&nalSize);
vp.buffer[0] = *(pNalSize + 3);
vp.buffer[1] = *(pNalSize + 2);
vp.buffer[2] = *(pNalSize + 1);
vp.buffer[3] = *(pNalSize);
CVPixelBufferRef pixelBuffer = NULL;
int nalType = vp.buffer[4] & 0x1F;
DDLogDebug(@"Nal type is %d",nalType);
switch (nalType) {
case 0x05:
//Nal type is IDR frame
if([self initH264Decoder]) {
pixelBuffer = [self decode:vp];
}
break;
case 0x07:
//Nal type is SPS
_spsSize = vp.size - 4;
_sps = malloc(_spsSize);
memcpy(_sps, vp.buffer + 4, _spsSize);
break;
case 0x08:
//Nal type is PPS
_ppsSize = vp.size - 4;
_pps = malloc(_ppsSize);
memcpy(_pps, vp.buffer + 4, _ppsSize);
break;
default:
//Nal type is B/P frame
pixelBuffer = [self decode:vp];
break;
}
if(pixelBuffer) {
dispatch_sync(dispatch_get_main_queue(), ^{
// 播放
//_glLayer.pixelBuffer = pixelBuffer;
// 对CVPixelBufferRef进行处理，获取RBG像素数据
[self writePixelBuffer:pixelBuffer];
// 后续操作……
});
CVPixelBufferRelease(pixelBuffer);
}
nalIndex++;
DDLogDebug(@"Read Nalu size %ld", (long)vp.size);
}
CFAbsoluteTime linkTime = (CFAbsoluteTimeGetCurrent() - startTime);
DDLogDebug(@"Ellapse %f ms,frame count:%ld", linkTime * 1000.0,(long)nalIndex);
DDLogDebug(@"Decode end");
[parser close];
[self clearH264Deocder];
}
其中，对VideoPacket进行解码：
-(CVPixelBufferRef)decode:(VideoPacket*)vp
{
CVPixelBufferRef outputPixelBuffer = NULL;
CMBlockBufferRef blockBuffer = NULL;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
(void*)vp.buffer, vp.size,
kCFAllocatorNull,
NULL, 0, vp.size,
0, &blockBuffer);
if(status == kCMBlockBufferNoErr) {
CMSampleBufferRef sampleBuffer = NULL;
const size_t sampleSizeArray[] = {vp.size};
status = CMSampleBufferCreateReady(kCFAllocatorDefault,
blockBuffer,
_decoderFormatDescription ,
1, 0, NULL, 1, sampleSizeArray,
&sampleBuffer);
if (status == kCMBlockBufferNoErr && sampleBuffer) {
VTDecodeFrameFlags flags = 0;
VTDecodeInfoFlags flagOut = 0;
OSStatus decodeStatus = VTDecompressionSessionDecodeFrame(_deocderSession,
sampleBuffer,
flags,
&outputPixelBuffer,
&flagOut);
if(decodeStatus == kVTInvalidSessionErr) {
DDLogDebug(@"IOS8VT: Invalid session, reset decoder session");
} else if(decodeStatus == kVTVideoDecoderBadDataErr) {
DDLogDebug(@"IOS8VT: decode failed status=%d(Bad data)", (int)decodeStatus);
} else if(decodeStatus != noErr) {
DDLogDebug(@"IOS8VT: decode failed status=%d", (int)decodeStatus);
}
CFRelease(sampleBuffer);
}
CFRelease(blockBuffer);
}
return outputPixelBuffer;
}
处理CVPixelBufferRef，获取RGB像素
在writePixelBuffer方法中，对CVPixelBufferRef进行处理，可得到RGB像素。Video Toolbox解码后的得到的图像数据并不能直接由CPU访问，需先用CVPixelBufferLockBaseAddress()锁定地址才能从主存访问，否则调用CVPixelBufferGetBaseAddressOfPlane等函数则返回NULL或无效值。值得注意的是，CVPixelBufferLockBaseAddress自身的调用并不消耗多少性能，一般情况，锁定之后，往CVPixelBuffer拷贝内存才是相对耗时的操作，比如计算内存偏移。待数据处理完毕，请记得使用CVPixelBufferUnlockBaseAddress() unlock地址。
¡
将void *的RGB数据强转为unsigned char *类型的images数据，以便用C语言代码进行后续处理：
- (void)writePixelBuffer:(CVPixelBufferRef)pixelBuffer
{
CVPixelBufferLockBaseAddress(pixelBuffer,kCVPixelBufferLock_ReadOnly);
void * rgb_data = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer,0);
images = (unsigned char *)rgb_data;
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
}
Clear
解码完成后，Clear H264解码器。
-(void)clearH264Deocder
{
if(_deocderSession) {
VTDecompressionSessionInvalidate(_deocderSession);
CFRelease(_deocderSession);
_deocderSession = NULL;
}
if(_decoderFormatDescription) {
CFRelease(_decoderFormatDescription);
_decoderFormatDescription = NULL;
}
free(_sps);
free(_pps);
_spsSize = _ppsSize = 0;
}
项目中的实例化变量定义
{
uint8_t *_sps;
NSInteger _spsSize;
uint8_t *_pps;
NSInteger _ppsSize;
VTDecompressionSessionRef _deocderSession;
CMVideoFormatDescriptionRef _decoderFormatDescription;
AAPLEAGLLayer *_glLayer;
unsigned char *images;
}