VTCompressionSession
要将相机采集到的数据进行编码，使用VideoToolBox的编码器进行编码
编码器作用就是将未压缩的CVPixelBuffer数据编码成压缩的CMSampleBuffer数据
未压缩的数据是CVPixelBuffer，压缩的数据是CMBlockBuffer,都是被封装在CMSampleBuffer里
压缩数据和未压缩数据的区别
image.png
编码器的作用如下
image.png
编码器使用的工作流如下：
image.png
1.创建编码器，需要提供的数据
image.png
函数
//OSStatus 返回创建的状态
OSStatus VTCompressionSessionCreate(
//1. 一个分配器 默认为Null
CFAllocatorRef allocator,
//2. 视频帧的像素宽度
int32_t width,
//3. 视频帧的像素高度
int32_t height,
//4. 编码类型 常用H264编码 kCMVideoCodecType_H264
CMVideoCodecType codecType,
//5. 编码方式Null由videoToolBox选择
CFDictionaryRef encoderSpecification,
//6. 创建一个像素缓冲池的属性 Null为由videoToolBox创建
CFDictionaryRef sourceImageBufferAttributes,
//7. 数据压缩分配器 默认为Null
CFAllocatorRef compressedDataAllocator,
//8. 输出回调 VTCompressionSessionEncodeFrame
VTCompressionOutputCallback outputCallback,
//9. 回调对象
void *outputCallbackRefCon,
//10. VTCompressionSession 要创建的对象
VTCompressionSessionRef _Nullable *compressionSessionOut
);
从Camera摄像头采集到的CMSampleBuffer容器数据,获取其包含的未编码的数据CVPixelBuffer(即CVImageBufferRef)，拿到宽高，并使用编码成kCMVideoCodecType_H264的基本数据Element Stream流，填充session
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
VTCompressionSessionRef session;
OSStatus ret = VTCompressionSessionCreate(NULL, (int)width, (int)height, kCMVideoCodecType_H264, NULL, NULL, NULL, OutputCallback, NULL, &session);
2.设置编码器的属性
image.png
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_High_AutoLevel)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse) // 不产生 B 帧
3.开始编码
VTCompressionSessionEncodeFrame开始编码，编码后的数据在回调进行处理
image.png
/*!
@function VTCompressionSessionEncodeFrame
@abstract
Call this function to present frames to the compression session.
Encoded frames may or may not be output before the function returns.
@discussion
The client should not modify the pixel data after making this call.
The session and/or encoder will retain the image buffer as long as necessary.
@param session
The compression session.
@param imageBuffer
A CVImageBuffer containing a video frame to be compressed.
Must have a nonzero reference count.
@param presentationTimeStamp
The presentation timestamp for this frame, to be attached to the sample buffer.
Each presentation timestamp passed to a session must be greater than the previous one.
@param duration
The presentation duration for this frame, to be attached to the sample buffer.
If you do not have duration information, pass kCMTimeInvalid.
@param frameProperties
Contains key/value pairs specifying additional properties for encoding this frame.
Note that some session properties may also be changed between frames.
Such changes have effect on subsequently encoded frames.
@param sourceFrameRefcon
Your reference value for the frame, which will be passed to the output callback function.
@param infoFlagsOut
Points to a VTEncodeInfoFlags to receive information about the encode operation.
The kVTEncodeInfo_Asynchronous bit may be set if the encode is (or was) running
asynchronously.
The kVTEncodeInfo_FrameDropped bit may be set if the frame was dropped (synchronously).
Pass NULL if you do not want to receive this information.
*/
VT_EXPORT OSStatus
VTCompressionSessionEncodeFrame(
CM_NONNULL VTCompressionSessionRef session,
CM_NONNULL CVImageBufferRef imageBuffer,
CMTime presentationTimeStamp,
CMTime duration, // may be kCMTimeInvalid
CM_NULLABLE CFDictionaryRef frameProperties,
void * CM_NULLABLE sourceFrameRefcon,
VTEncodeInfoFlags * CM_NULLABLE infoFlagsOut )
PTS: 即presentationTimestamp，每一次传入的一定比前面一帧的要大，编码后被依附到CVSampleBuffer里，也可以不设置
let currentTimeMills = Int64(CFAbsoluteTimeGetCurrent() * 1000)
if self.encodingTimeMills == -1 {
self.encodingTimeMills = currentTimeMills
}
let encodingDuration = currentTimeMills - self.encodingTimeMills
let presentationTimestamp = CMTimeMake(value: encodingDuration, timescale: 1000) // 当前编码视频帧的时间戳，单位为毫秒
VTCompressionSessionEncodeFrame(session, imageBuffer, presentationTimestamp, kCMTimeInvalid, NULL, NULL, NULL);
编码的回调处理
编码后，得到的是压缩的数据CMSampleBuffer,
回调处理
void OutputCallback(void *outputCallbackRefCon,
void *sourceFrameRefCon,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CMSampleBufferRef sampleBuffer)
要对压缩数据CMSampleBuffer进行处理.
image.png
压缩后的数据CMSampleBuffer, 是MEPEG-4形式，需要转为Elementary Stream形式，再在下面两处地方使用
一般有两种类型的使用，一是网络流，另一个是写到文件，他们都是接受Elementary Stream形式的数据，在回调里，要进行相应的处理，转为Elementary Stream形式
以写入到文件.mov为例,
一个CMSampleBuffer只能是一帧数据，但一帧可能有多个NALU单元，所以需要循环的把NALU转到NSData里,一个NALU由startcode码+type类型+数据组成
image.png
在抽取NALU时，因为CMSampleBuffer里的数据是MPEG-4形式的，即含有一个四个字节大小的header，header里存储的是NALU数据的大小，要进行相应的转化
image.png
另外，对于关键帧I帧，还需要抽取sps，pps，转到NSData里
image.png
void encodeOutputDataCallback(void * CM_NULLABLE outputCallbackRefCon, void * CM_NULLABLE sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CM_NULLABLE CMSampleBufferRef sampleBuffer)
{
if (noErr != status || nil == sampleBuffer)
{
NSLog(@"VEVideoEncoder::encodeOutputCallback Error : %d!", (int)status);
return;
}
if (nil == outputCallbackRefCon)
{
return;
}
if (!CMSampleBufferDataIsReady(sampleBuffer))
{
return;
}
if (infoFlags & kVTEncodeInfo_FrameDropped)
{
NSLog(@"VEVideoEncoder::H264 encode dropped frame.");
return;
}
VEVideoEncoder *encoder = (__bridge VEVideoEncoder *)outputCallbackRefCon;
const char header[] = "\x00\x00\x00\x01";
size_t headerLen = (sizeof header) - 1;
NSData *headerData = [NSData dataWithBytes:header length:headerLen];
// 判断是否是关键帧
bool isKeyFrame = !CFDictionaryContainsKey((CFDictionaryRef)CFArrayGetValueAtIndex(CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true), 0), (const void *)kCMSampleAttachmentKey_NotSync);
if (isKeyFrame)
{
NSLog(@"VEVideoEncoder::编码了一个关键帧");
CMFormatDescriptionRef formatDescriptionRef = CMSampleBufferGetFormatDescription(sampleBuffer);
// 关键帧需要加上SPS、PPS信息
size_t sParameterSetSize, sParameterSetCount;
const uint8_t *sParameterSet;
OSStatus spsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDescriptionRef, 0, &sParameterSet, &sParameterSetSize, &sParameterSetCount, 0);
size_t pParameterSetSize, pParameterSetCount;
const uint8_t *pParameterSet;
OSStatus ppsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDescriptionRef, 1, &pParameterSet, &pParameterSetSize, &pParameterSetCount, 0);
if (noErr == spsStatus && noErr == ppsStatus)
{
NSData *sps = [NSData dataWithBytes:sParameterSet length:sParameterSetSize];
NSData *pps = [NSData dataWithBytes:pParameterSet length:pParameterSetSize];
NSMutableData *spsData = [NSMutableData data];
[spsData appendData:headerData];
[spsData appendData:sps];
if ([encoder.delegate respondsToSelector:@selector(videoEncodeOutputDataCallback:isKeyFrame:)])
{
[encoder.delegate videoEncodeOutputDataCallback:spsData isKeyFrame:isKeyFrame];
}
NSMutableData *ppsData = [NSMutableData data];
[ppsData appendData:headerData];
[ppsData appendData:pps];
if ([encoder.delegate respondsToSelector:@selector(videoEncodeOutputDataCallback:isKeyFrame:)])
{
[encoder.delegate videoEncodeOutputDataCallback:ppsData isKeyFrame:isKeyFrame];
}
}
}
CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t length, totalLength;
char *dataPointer;
status = CMBlockBufferGetDataPointer(blockBuffer, 0, &length, &totalLength, &dataPointer);
if (noErr != status)
{
NSLog(@"VEVideoEncoder::CMBlockBufferGetDataPointer Error : %d!", (int)status);
return;
}
size_t bufferOffset = 0;
static const int avcHeaderLength = 4;
while (bufferOffset < totalLength - avcHeaderLength)
{
// 读取 NAL 单元长度
uint32_t nalUnitLength = 0;
memcpy(&nalUnitLength, dataPointer + bufferOffset, avcHeaderLength);
// 网络上传输数据普遍采用的都是大端
// 大端转小端
nalUnitLength = CFSwapInt32BigToHost(nalUnitLength);
NSData *frameData = [[NSData alloc] initWithBytes:(dataPointer + bufferOffset + avcHeaderLength) length:nalUnitLength];
NSMutableData *outputFrameData = [NSMutableData data];
[outputFrameData appendData:headerData];
[outputFrameData appendData:frameData];
bufferOffset += avcHeaderLength + nalUnitLength;
if ([encoder.delegate respondsToSelector:@selector(videoEncodeOutputDataCallback:isKeyFrame:)])
{
[encoder.delegate videoEncodeOutputDataCallback:outputFrameData isKeyFrame:isKeyFrame];
}
}
}
4.完成编码
VTCompressionSessionCompleteFrames(session, completeUntilPresentationTimeStamp);
5.销毁编码器
if (session) {
VTCompressionSessionInvalidate(session);
CFRelease(session);
}