音视频-OpenGL ES渲染视频图像

这篇文章处理的是编码后的视频图像数据CVPixelBufferRef，我们使用CAEAGLLayer来作为图像的渲染图层。
该功能使用到的是OpenGL ES的相关技术，如果对该技术不了解的伙伴，请先了解相关技术。

一、着色器

1.1顶点着色器：

const GLchar *shader_vsh = (const GLchar*)"attribute vec4 position;"
"attribute vec2 texCoord;"
"uniform float preferredRotation;"
"varying vec2 texCoordVarying;"
"void main()"
"{"
"    mat4 rotationMatrix = mat4(cos(preferredRotation), -sin(preferredRotation), 0.0, 0.0,"
"                               sin(preferredRotation),  cos(preferredRotation), 0.0, 0.0,"
"                               0.0,                        0.0, 1.0, 0.0,"
"                               0.0,                        0.0, 0.0, 1.0);"
"    gl_Position = position * rotationMatrix;"
"    texCoordVarying = texCoord;"
"}";

• "attribute vec4 position：顶点坐标
  attribute vec2 texCoord：纹理坐标
  uniform float preferredRotation：旋转角度
  attribute vec2 texCoordVarying：用于把texCoord（纹理坐标）传递到片元着器。
  mat4 rotationMatrix4x4 的旋转矩阵
  gl_Position：内建变量，用来接收顶点坐标。

1.1片元着色器：

const GLchar *shader_fsh = (const GLchar*)"varying highp vec2 texCoordVarying;"
"precision mediump float;"
"uniform sampler2D SamplerY;"
"uniform sampler2D SamplerUV;"
"uniform mat3 colorConversionMatrix;"
"void main()"
"{"
"    mediump vec3 yuv;"
"    lowp vec3 rgb;"
//   Subtract constants to map the video range start at 0
"    yuv.x = (texture2D(SamplerY, texCoordVarying).r - (16.0/255.0));"
"    yuv.yz = (texture2D(SamplerUV, texCoordVarying).rg - vec2(0.5, 0.5));"
"    rgb = colorConversionMatrix * yuv;"
"    gl_FragColor = vec4(rgb, 1);"
"}";

• varying highp vec2 texCoordVarying：从顶点着色器传递过来的纹理坐标。
  precision mediump float;：精度设置，中间精度float。
  uniform sampler2D SamplerY;：Y亮度纹理采样器。
  uniform sampler2D SamplerUV;：UV色度纹理采样器。
  uniform mat3 colorConversionMatrix;：颜色转换矩阵。
  gl_FragColor：内建函数，输出的颜色用于随后的像素操作。

二、初始化

2.1继承之CAEAGLLayer自定义类CQEAGLLayer。

 CGSize size = UIScreen.mainScreen.bounds.size;
_displayLayer = [[CQEAGLLayer alloc] initWithFrame:CGRectMake(size.width*0.5, 100, size.width*0.5, size.height*0.5)];
[self.view.layer addSublayer:_displayLayer];

2.2初始化代码：

- (instancetype)initWithFrame:(CGRect)frame {
    self = [super init];
    if (self) {
        CGFloat scale = [[UIScreen mainScreen] scale];
        self.contentsScale = scale;
        //一个布尔值，指示层是否包含完全不透明的内容.默认为NO
        self.opaque = TRUE;
        //kEAGLDrawablePropertyRetainedBacking指定可绘制表面在显示后是否保留其内容的键.默认为NO.
        self.drawableProperties = @{ kEAGLDrawablePropertyRetainedBacking :[NSNumber numberWithBool:YES]};
        [self setFrame:frame];
        // 1、设置绘制框架的上下文.
        _context = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
        
        if (!_context || ![EAGLContext setCurrentContext:_context]) {
            return nil;
        }
        // 将默认转换设置为BT.709，这是HDTV的标准
        _preferredConversion = kColorConversion709;
        
        //2、设置缓冲区
        [self setupBuffers];
        //3、加载shaders 着色器
        [self loadShaders];
        //4、使用着色程序
        glUseProgram(self.program);
        
        // 0 和 1 分别代表_lumaTexture 和 _chromaTexture的纹理 ID。
        glUniform1i(uniforms[UNIFORM_Y], 0);
        glUniform1i(uniforms[UNIFORM_UV], 1);
        glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
        glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
    }
    return self;
}

• 1、创建上下文。
• 2、设置缓冲区。
• 3、使用program

2.3 设置缓冲区

- (void)setupBuffers {
    glDisable(GL_DEPTH_TEST);
    //默认顶点属性(position)是关闭的,所以使用前要手动打开
    glEnableVertexAttribArray(ATTRIB_VERTEX);
    glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
    //打开纹理数据读取通道
    glEnableVertexAttribArray(ATTRIB_TEXCOORD);
    glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
    //创建buffer
    [self createBuffers];
}

- (void) createBuffers {
    //创建帧缓存区 frameBuffer
    glGenFramebuffers(1, &_frameBufferHandle);
    glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
    
    //创建颜色缓存区 RenderBuffer
    glGenRenderbuffers(1, &_colorBufferHandle);
    glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
    
    //绑定渲染缓存区
    [_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:self];
    
    //设置渲染缓存区的尺寸
    glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &_backingWidth);
    glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &_backingHeight);
    
    //绑定renderBuffer到FrameBuffer
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _colorBufferHandle);
    
    //检查FrameBuffer状态
    if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
        NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
    }
}

• glVertexAttribPointer顶点数据解析方式:
  参数1: 指定从索引0开始取数据，与顶点着色器对应
  参数2: 顶点属性大小
  参数3: 数据类型
  参数4: 归一化
  参数5: 步长（Stride)
  参数6: 数据在缓冲区起始位置的偏移量

2.4加载着色器

- (BOOL)loadShaders {
    GLuint vertShader = 0, fragShader = 0;
    // 创建着色program.
    self.program = glCreateProgram();
    //编译顶点着色器
    if(![self compileShaderString:&vertShader type:GL_VERTEX_SHADER shaderString:shader_vsh]) {
        NSLog(@"Failed to compile vertex shader");
        return NO;
    }
    //编译片元着色器
    if(![self compileShaderString:&fragShader type:GL_FRAGMENT_SHADER shaderString:shader_fsh]) {
        NSLog(@"Failed to compile fragment shader");
        return NO;
    }
    
    // 附着顶点着色器到program.
    glAttachShader(self.program, vertShader);
    // 附着片元着色器到program.
    glAttachShader(self.program, fragShader);
    
    // 绑定属性位置。这需要在链接之前完成.(让ATTRIB_VERTEX/ATTRIB_TEXCOORD 与position/texCoord产生连接)
    glBindAttribLocation(self.program, ATTRIB_VERTEX, "position");
    glBindAttribLocation(self.program, ATTRIB_TEXCOORD, "texCoord");
    
    // Link the program.
    if (![self linkProgram:self.program]) {
        NSLog(@"Failed to link program: %d", self.program);
        
        if (vertShader) {
            glDeleteShader(vertShader);
            vertShader = 0;
        }
        if (fragShader) {
            glDeleteShader(fragShader);
            fragShader = 0;
        }
        if (self.program) {
            glDeleteProgram(self.program);
            self.program = 0;
        }
        
        return NO;
    }
    
    //获取uniform的位置
    //Y亮度纹理
    uniforms[UNIFORM_Y] = glGetUniformLocation(self.program, "SamplerY");
    //UV色量纹理
    uniforms[UNIFORM_UV] = glGetUniformLocation(self.program, "SamplerUV");
    //旋转角度preferredRotation
    uniforms[UNIFORM_ROTATION_ANGLE] = glGetUniformLocation(self.program, "preferredRotation");
    //YUV->RGB
    uniforms[UNIFORM_COLOR_CONVERSION_MATRIX] = glGetUniformLocation(self.program, "colorConversionMatrix");
    
    // Release vertex and fragment shaders.
    if (vertShader) {
        glDetachShader(self.program, vertShader);
        glDeleteShader(vertShader);
    }
    if (fragShader) {
        glDetachShader(self.program, fragShader);
        glDeleteShader(fragShader);
    }
    
    return YES;
}

2.5编译

- (BOOL)compileShaderString:(GLuint *)shader type:(GLenum)type shaderString:(const GLchar*)shaderString
{
    *shader = glCreateShader(type);
    glShaderSource(*shader, 1, &shaderString, NULL);
    glCompileShader(*shader);
    
#if defined(DEBUG)
    GLint logLength;
    glGetShaderiv(*shader, GL_INFO_LOG_LENGTH, &logLength);
    if (logLength > 0) {
        GLchar *log = (GLchar *)malloc(logLength);
        glGetShaderInfoLog(*shader, logLength, &logLength, log);
        NSLog(@"Shader compile log:\n%s", log);
        free(log);
    }
#endif
    GLint status = 0;
    glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
    if (status == 0) {
        glDeleteShader(*shader);
        return NO;
    }
    
    return YES;
}

2.5链接

- (BOOL)linkProgram:(GLuint)prog {
    GLint status;
    glLinkProgram(prog);
    
#if defined(DEBUG)
    GLint logLength;
    glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength);
    if (logLength > 0) {
        GLchar *log = (GLchar *)malloc(logLength);
        glGetProgramInfoLog(prog, logLength, &logLength, log);
        NSLog(@"Program link log:\n%s", log);
        free(log);
    }
#endif
    
    glGetProgramiv(prog, GL_LINK_STATUS, &status);
    if (status == 0) {
        return NO;
    }
    return YES;
}

三、绘制

3.1设置CVPixelBufferRef

- (void)setPixelBuffer:(CVPixelBufferRef)pb {
    if(_pixelBuffer) {
        CVPixelBufferRelease(_pixelBuffer);
    }
    _pixelBuffer = CVPixelBufferRetain(pb);
    
    //获取视频帧的宽与高
    int frameWidth = (int)CVPixelBufferGetWidth(_pixelBuffer);
    int frameHeight = (int)CVPixelBufferGetHeight(_pixelBuffer);
    
    //显示_pixelBuffer
    [self display:_pixelBuffer width:frameWidth height:frameHeight];
}

• 在iOS里，我们经常能看到 CVPixelBufferRef 这个类型，在Camera 采集返回的数据里得到一个CMSampleBufferRef，而每个CMSampleBufferRef里则包含一个 CVPixelBufferRef，在视频硬解码的返回数据里也是一个 CVPixelBufferRef（里面包好了所有的压缩的图片信息）。CVPixelBufferRef：是一种像素图片类型，由于CV开头，所以它是属于 CoreVideo 模块的。

3.2显示

- (void)display:(CVPixelBufferRef)pixelBuffer width:(uint32_t)frameWidth height:(uint32_t)frameHeight {
    if (!_context || ![EAGLContext setCurrentContext:_context]) {
        return;
    }
    if(pixelBuffer == NULL) {
        NSLog(@"PixelBuffer is null");
        return;
    }
    
    CVReturn errer;
    size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
    CFTypeRef colorAttachments = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
    if (CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) {
        _preferredConversion = kColorConversion601;
    }
    else {
        _preferredConversion = kColorConversion709;
    }
    
    CVOpenGLESTextureCacheRef _videoTextureCache;
  //创建 CVOpenGLESTextureCacheRef 创建新的纹理缓存
    errer = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _context, NULL, &_videoTextureCache);
    if (error != noErr) {
        NSLog(@"CVOpenGLESTextureCacheCreate error : %d", errer);
        return;
    }
    glActiveTexture(GL_TEXTURE0); //激活纹理
    //1.创建Y纹理(亮度纹理)
    errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
                                                       _videoTextureCache,
                                                       pixelBuffer,
                                                       NULL,
                                                       GL_TEXTURE_2D,
                                                       GL_RED_EXT,
                                                       frameWidth,
                                                       frameHeight,
                                                       GL_RED_EXT,
                                                       GL_UNSIGNED_BYTE,
                                                       0,
                                                       &_lumaTexture);
    if (errer) {
        NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", errer);
    }
    
    //2.配置Y亮度纹理属性
    //绑定纹理.
    glBindTexture(CVOpenGLESTextureGetTarget(_lumaTexture), CVOpenGLESTextureGetName(_lumaTexture));
    //配置纹理放大/缩小过滤方式以及纹理围绕S/T环绕方式
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    
    //3.UV-plane 纹理
    //如果颜色通道个数>1,则除了Y还有UV-Plane.
    if(planeCount == 2) {
        // UV-plane.
        glActiveTexture(GL_TEXTURE1); //激活UV-plane纹理
        //4.创建UV-plane纹理
        errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
                                                           _videoTextureCache,
                                                           pixelBuffer,
                                                           NULL,
                                                           GL_TEXTURE_2D,
                                                           GL_RG_EXT,
                                                           frameWidth / 2,
                                                           frameHeight / 2,
                                                           GL_RG_EXT,
                                                           GL_UNSIGNED_BYTE,
                                                           1,
                                                           &_chromaTexture);
        if (error) {
            NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", error);
        }
        
        //5.绑定纹理
        glBindTexture(CVOpenGLESTextureGetTarget(_chromaTexture), CVOpenGLESTextureGetName(_chromaTexture));
        //6.配置纹理放大/缩小过滤方式以及纹理围绕S/T环绕方式
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    }
    
    //绑定帧缓存区
    glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
    //设置视口.
    glViewport(0, 0, _backingWidth, _backingHeight);
    //清理屏幕
    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT);
    //使用shaderProgram
    glUseProgram(self.program);
    //传递Uniform属性到shader
    //UNIFORM_ROTATION_ANGLE 旋转角度
    glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
    //UNIFORM_COLOR_CONVERSION_MATRIX YUV->RGB颜色矩阵
    glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
    
    // 根据视频的方向和纵横比设置四边形顶点
    CGRect viewBounds = self.bounds;
    CGSize contentSize = CGSizeMake(frameWidth, frameHeight);
    
    /*
     AVMakeRectWithAspectRatioInsideRect
     功能: 返回一个按比例缩放的CGRect，该CGRect保持由边界CGRect内的CGSize指定的纵横比
     参数1:希望保持的宽高比或纵横比
     参数2:填充的rect
     */
    CGRect vertexSamplingRect = AVMakeRectWithAspectRatioInsideRect(contentSize, viewBounds);
    
    // 计算标准化的四边形坐标以将帧绘制到其中
    //标准化采样大小
    CGSize normalizedSamplingSize = CGSizeMake(0.0, 0.0);
    //标准化规模
    CGSize cropScaleAmount = CGSizeMake(vertexSamplingRect.size.width/viewBounds.size.width,vertexSamplingRect.size.height/viewBounds.size.height);
    
    // 规范化四元顶点
    if (cropScaleAmount.width > cropScaleAmount.height) {
        normalizedSamplingSize.width = 1.0;
        normalizedSamplingSize.height = cropScaleAmount.height/cropScaleAmount.width;
    }
    else {
        normalizedSamplingSize.width = cropScaleAmount.width/cropScaleAmount.height;
        normalizedSamplingSize.height = 1.0;;
    }
    
    /*
     四顶点数据定义了我们绘制像素缓冲区的二维平面区域。
     使用（-1，-1）和（1,1）分别作为左下角和右上角坐标形成的顶点数据覆盖整个屏幕。
     */
    GLfloat quadVertexData [] = {
        -1 * normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
        normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
        -1 * normalizedSamplingSize.width, normalizedSamplingSize.height,
        normalizedSamplingSize.width, normalizedSamplingSize.height,
    };
    
    // 更新属性值.
    //坐标数据
    glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, quadVertexData);
    glEnableVertexAttribArray(ATTRIB_VERTEX);
    
    /*
     纹理顶点的设置使我们垂直翻转纹理。这使得我们的左上角原点缓冲区匹配OpenGL的左下角纹理坐标系
     */
    CGRect textureSamplingRect = CGRectMake(0, 0, 1, 1);
    GLfloat quadTextureData[] =  {
        CGRectGetMinX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
        CGRectGetMaxX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
        CGRectGetMinX(textureSamplingRect), CGRectGetMinY(textureSamplingRect),
        CGRectGetMaxX(textureSamplingRect), CGRectGetMinY(textureSamplingRect)
    };
    
    //更新纹理坐标属性值
    glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, 0, 0, quadTextureData);
    glEnableVertexAttribArray(ATTRIB_TEXCOORD);
    
    //绘制图形
    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
    
    //绑定渲染缓存区->显示到屏幕
    glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
    [_context presentRenderbuffer:GL_RENDERBUFFER];
    
    //清理纹理,方便下一帧纹理显示
    [self cleanUpTextures];
    // 定期纹理缓存刷新每帧
    CVOpenGLESTextureCacheFlush(_videoTextureCache, 0);
    
    if(_videoTextureCache) {
        CFRelease(_videoTextureCache);
    }
}

• CVBufferGetAttachment使用像素缓冲区的颜色附件确定适当的颜色转换矩阵:
  参数1: 像素缓存区
  参数2: kCVImageBufferYCbCrMatrixKey YCbCr->RGB
  参数3: 附件模式,NULL
• CVOpenGLESTextureCacheCreate创建新的纹理缓存：
  参数1: kCFAllocatorDefault默认内存分配器.
  参数2: NULL
  参数3: EAGLContext 图形上下文
  参数4: NULL
  参数5: 新创建的纹理缓存
• CVOpenGLESTextureCacheCreateTextureFromImage创建亮度/色度纹理-Y/UV纹理:
  参数1: 内存分配器,kCFAllocatorDefault
  参数2: 纹理缓存.纹理缓存将管理纹理的纹理缓存对象
  参数3: sourceImage.
  参数4: 纹理属性.默认给NULL
  参数5: 目标纹理,GL_TEXTURE_2D
  参数6: 指定纹理中颜色组件的数量(GL_RGBA, GL_LUMINANCE, GL_RGBA8_OES, GL_RG, and GL_RED (NOTE: 在 GLES3 使用 GL_R8 替代 GL_RED).)
  参数7: 帧宽度
  参数8: 帧高度
  参数9: 格式指定像素数据的格式
  参数10: 指定像素数据的数据类型,GL_UNSIGNED_BYTE
  参数11: planeIndex
  参数12: 纹理输出新创建的纹理对象将放置在此处。