39.FFmpeg学习笔记 - iOS硬编码视频到h264

一、概述

硬编码使用场景通常是从摄像头采集图像数据，然后利用系统提供的接口进行硬编码。

从摄像头采集数据用的是AVFoundation框架，硬编码用的是iOS 8开放的VideoToolbox框架。

二、用AVFoundation采集数据

用AVFoundation采集摄像头的数据步骤基本固定，下面详细说一说：

1.初始化会话，并设置摄像头采集图像的分辨率

    //初始化会话
    self.captureSession = [[AVCaptureSession alloc] init];
    if ([self.captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) {
        self.captureSession.sessionPreset = AVCaptureSessionPreset1280x720;
    }

2.查找摄像头，并添加到会话中

 //查找后置摄像头
    AVCaptureDevice *inputCamera = nil;
    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
    for (AVCaptureDevice *device in devices)
    {
        if ([device position] == AVCaptureDevicePositionBack)
        {
            inputCamera = device;
        }
    }
    //包装到AVCaptureDeviceInput
    NSError *error;
    self.deviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:inputCamera error:&error];
    if (error) {
        NSLog(@"%@", error);
        return;
    }
    
    //输入设备添加到会话
    if ([self.captureSession canAddInput:self.deviceInput]) {
        [self.captureSession addInput:self.deviceInput];
    }

3.创建输出设备，并设置输出数据的属性，然后添加到会话中

    //创建输出设备
    self.captureOutput = [[AVCaptureVideoDataOutput alloc] init];
    //允许丢帧
    self.captureOutput.alwaysDiscardsLateVideoFrames = YES;
    
    //打印支持的像素格式
    NSArray *typeArray = [self.captureOutput availableVideoCVPixelFormatTypes];
    for (NSNumber *type in typeArray) {
        NSLog(@"**%c%c%c%c", (type.intValue >> 24), ((type.intValue & 0x00ff0000) >> 16), ((type.intValue & 0x0000ff00) >> 8), (type.intValue & 0x000000ff));
    }
    //设置输出的像素格式
    [self.captureOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];

     //设置输出串行队列和数据回调
    dispatch_queue_t outputQueue = dispatch_queue_create("CaptureQueue", DISPATCH_QUEUE_SERIAL);
    [self.captureOutput setSampleBufferDelegate:self queue:outputQueue];
    
    if ([self.captureSession canAddOutput:self.captureOutput]) {
        [self.captureSession addOutput:self.captureOutput];
    }

4.初始化连接，并设置输出视频的方向。如果是前置摄像头，需要设置镜像属性videoMirrored，否则输出的视频是左右镜像的。另外需要注意的是，设置此属性，仅影响输出的数据，并不能改变实时预览的效果。

    //设置输出的视频方向
    self.captureConnection = [self.captureOutput connectionWithMediaType:AVMediaTypeVideo];
    [self.captureConnection setVideoOrientation:AVCaptureVideoOrientationPortrait];
    if (inputCamera.position == AVCaptureDevicePositionFront && self.captureConnection.supportsVideoMirroring)
    {
        self.captureConnection.videoMirrored = YES;
    }

5.设置预览图层，用于实时显示摄像头采集到底画面。

    self.previewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:self.captureSession];
    [self.previewLayer setVideoGravity:AVLayerVideoGravityResizeAspect];
    //设置实时预览的方向
    self.previewLayer.connection.videoOrientation = AVCaptureVideoOrientationPortrait;
    [self.previewLayer setFrame:self.view.bounds];
    [self.view.layer addSublayer:self.previewLayer];

6.一切准备就绪，调用如下代码，开始视频采集。

    // 摄像头权限判断
    AVAuthorizationStatus videoAuthStatus = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
    if (videoAuthStatus != AVAuthorizationStatusAuthorized)
    {
        NSLog(@"摄像头权限没开");
        return;
    }
    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
        [self.captureSession startRunning];
    });

由于startRunning的调用会block线程，所以进行了异步调用。

7.实现代理回调函数，每采集到一帧画面，都会调用此函数。然后用回调的视频数据进行编码。

- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {

}

三、用VideoToolbox编码

一些数据结构：

• CVPixelBufferRef/CVImageBufferRef：存放编码前和解码后的图像数据，这俩货其实是同一个东西
• CMTime：时间戳相关，时间以64-bit/32-bit的形式出现
• CMBlockBufferRef：编码后输出的数据
• CMFormatDescriptionRef/CMVideoFormatDescriptionRef：图像存储方式，编解码器等格式描述。这俩货也是同一个东西
• CMSampleBufferRef：存放编解码前后的视频图像的容器数据

1.先调用VTCompressionSessionCreate创建编码的session，宽高为视频采集到的宽高，编码类型选择h264，设置encodeCallback回调函数，每编码一帧会调用这个函数。_compressionSession为创建的session。

    //宽高和视频采集的分辨率有关
    int width = 720, height = 1280;
    OSStatus status = VTCompressionSessionCreate(NULL, width, height, kCMVideoCodecType_H264, NULL, NULL, NULL, encodeCallback, (__bridge void *)(self),  &_compressionSession);
    if (noErr != status)
    {
        NSLog(@"could not create CompressionSession");
        return;
    }

2.创建session后，设置session的属性

    // 设置实时编码输出（避免延迟）
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
    if (status != noErr) {
        NSLog(@"set realtime fail");
        return;
    }

    //Baseline没有B帧，用于直播，减小延迟
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
    if (status != noErr) {
        NSLog(@"set profile level fail");
        return;
    }
    
    // GOP size
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(15));
    if (status != noErr) {
        NSLog(@"set gop size fail");
        return;
    }
    
    // 设置期望帧率
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(25));
    if (status != noErr) {
        NSLog(@"set ExpectedFrameRate fail");
        return;
    }
    
    //配置是否产生B帧
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
    if (noErr != status)
    {
        NSLog(@"set b frame allow fail");
        return;
    }
    
    //设置平均码率 单位 bit per second，3500Kbps
    int averageBitRate = 3500 * 1024;
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate));
    if (noErr != status)
    {
        NSLog(@"set AverageBitRate fail");
        return;
    }
    
    //参考webRTC 限制最大码率不超过平均码率的1.5倍，单位为byte per second
    int64_t dataLimitBytesPerSecondValue = averageBitRate * 1.5 / 8;
    CFNumberRef bytesPerSecond = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
    int64_t oneSecondValue = 1;
    CFNumberRef oneSecond = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
    const void* nums[2] = {bytesPerSecond, oneSecond};
    CFArrayRef dataRateLimits = CFArrayCreate(NULL, nums, 2, &kCFTypeArrayCallBacks);
    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
    if (noErr != status)
    {
        NSLog(@"set DataRateLimits fail");
        return;
    }
    

平均码率，可以参考h264文档中的建议码率，然后微调它，以此选择合适的码率。

视频大小	分辨率	建议码率
480P	720X480	1800Kbps
720P	1280X720	3500Kbps
1080P	1920X1080	8500Kbps

3.设置属性后，调用下面的一行代码，准备编码

    //准备编码
    VTCompressionSessionPrepareToEncodeFrames(self.compressionSession);

4.由于打算将编码的视频数据写入到文件中，所以下面创建文件用于写入

    //编码后，写入h264文件中
    NSString *file = [[NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES) lastObject] stringByAppendingPathComponent:@"result.h264"];
    [[NSFileManager defaultManager] removeItemAtPath:file error:nil];
    [[NSFileManager defaultManager] createFileAtPath:file contents:nil attributes:nil];
    self.fileHandle = [NSFileHandle fileHandleForWritingAtPath:file];

5.在视频采集的回调函数中调用VTCompressionSessionEncodeFrame来进行编码，执行后，会调用编码回调函数。

- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
    
    CVImageBufferRef imageBuffer = (CVImageBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
    //设置编码时间戳
    CMTime pts = CMTimeMake(self.frameNum++, 25);
    OSStatus status = VTCompressionSessionEncodeFrame(self.compressionSession,
                                                          imageBuffer,
                                                          pts,
                                                          kCMTimeInvalid,
                                                          NULL, NULL, NULL);
    if (status != noErr) {
        NSLog(@"Encode failed");
        VTCompressionSessionInvalidate(self.compressionSession);
        CFRelease(self.compressionSession);
        self.compressionSession = NULL;
        return;
    }
    NSLog(@"EncodeFrame Success");

}

6.在编码回调函数中，将得到的编码数据，写入到h264文件中。得到的编码数据是一串nalu单元，每个nalu单元之间用4个字节表示其后的nalu的长度。

在回调函数中，先判断此帧是否为关键帧，如果是关键帧，则获取sps、pps数据，并写入到文件中，然后将帧数据写入到文件中。

写入到h264文件中，需要将这4个字节换成固定4个字节的startcode：0x00000001，这样才能区分每个nalu单元。

void encodeCallback(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer)
{
    NSLog(@"encodeCallback");
    if (status != noErr) {
        return;
    }
    
    if (!CMSampleBufferDataIsReady(sampleBuffer))
    {
        return;
    }
    
    if (infoFlags & kVTEncodeInfo_FrameDropped)
    {
        NSLog(@"---encode dropped frame");
        return;
    }
    
    ViewController *encoder = (__bridge ViewController *)outputCallbackRefCon;

    const char header[] = "\x00\x00\x00\x01";
    size_t headerLen = 4;  //4字节的0x00000001分隔码
    NSData *headerData = [NSData dataWithBytes:header length:headerLen];
    
    // 判断是否是关键帧
    bool isKeyFrame = !CFDictionaryContainsKey((CFDictionaryRef)CFArrayGetValueAtIndex(CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true), 0), (const void *)kCMSampleAttachmentKey_NotSync);
    // 获取sps & pps数据
    if (isKeyFrame)
    {
        NSLog(@"编码了一个关键帧");
        CMFormatDescriptionRef formatDescriptionRef = CMSampleBufferGetFormatDescription(sampleBuffer);
        
        // 关键帧需要加上SPS、PPS信息
        size_t sParameterSetSize, sParameterSetCount;
        const uint8_t *sParameterSet;
        OSStatus spsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDescriptionRef, 0, &sParameterSet, &sParameterSetSize, &sParameterSetCount, 0);
        
        size_t pParameterSetSize, pParameterSetCount;
        const uint8_t *pParameterSet;
        OSStatus ppsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDescriptionRef, 1, &pParameterSet, &pParameterSetSize, &pParameterSetCount, 0);
        
        if (noErr == spsStatus && noErr == ppsStatus)
        {
            NSData *sps = [NSData dataWithBytes:sParameterSet length:sParameterSetSize];
            NSData *pps = [NSData dataWithBytes:pParameterSet length:pParameterSetSize];
            
            [encoder.fileHandle writeData:headerData];
            [encoder.fileHandle writeData:sps];
            [encoder.fileHandle writeData:headerData];
            [encoder.fileHandle writeData:pps];
         
        }

    }
    
    CMBlockBufferRef dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
    size_t length, totalLength;
    char *dataPointer;
    OSStatus statusCodeRet = CMBlockBufferGetDataPointer(dataBuffer, 0, &length, &totalLength, &dataPointer);
    if (statusCodeRet == noErr) {
        size_t bufferOffset = 0;
        static const int AVCCHeaderLength = 4; // 返回的nalu数据前四个字节不是0001的startcode，而是大端模式的帧长度length
        
        // 循环获取nalu数据
        while (bufferOffset < totalLength - AVCCHeaderLength) {
            uint32_t NALUnitLength = 0;
            // Read the NAL unit length
            memcpy(&NALUnitLength, dataPointer + bufferOffset, AVCCHeaderLength);
            
            // 从大端转系统端
            NALUnitLength = CFSwapInt32BigToHost(NALUnitLength);
            
            NSData* data = [[NSData alloc] initWithBytes:(dataPointer + bufferOffset + AVCCHeaderLength) length:NALUnitLength];
            [encoder.fileHandle writeData:headerData];
            [encoder.fileHandle writeData:data];
            
            // Move to the next NAL unit in the block buffer
            bufferOffset += AVCCHeaderLength + NALUnitLength;
        }
    }
}

 demo地址：

https://github.com/whoyouare888/Note/tree/master/VideoToolbox/VTEncode