解码
4个动作
- 采集-视频源数据CVPixelBuffer
- 编码(硬编码VideoToolBox)--NALU数据-CMBlockBuffer
- 编码(硬编码VideoToolBox)--NALU数据-CVPiexBuffer
- 将数据显示到屏幕上(渲染!)-OpenGL ES-- 视频渲染动作
解码思路 - 解析数据(NALU unit) I/P/B
- 初始化编码器
- 将解析后的H264, NALU，Unit输入编码器
- 解码完成后回调，输出解码数据
- 解码数据显示(OpenGL, ES)
解码三个核心函数：
- 创建session， VTDdcompressionSessionCreate
- 解码一个frame，VTDdcompressionSessionCreateFrame
- 销毁解码session，VTDdcompressionSessionCreateInvalidate
H264原始码流--> NALU
I帧：保留了一张完整的视频帧，解码关键
P帧：先前参考帧，差异数据，解码需要依赖I帧
B帧：双向参考帧，解码时既需要I帧，也需要P帧
如果H264码流中I帧丢失/错误，就会导致错误传递，P/B帧单独是完成不了解码工作！，花屏的现象产生。VideoToolBox硬编码编码H264帧，I帧，手动加入SPS/PPS
#include
#include
@interface AAPLEAGLLayer : CAEAGLLayer
@property CVPixelBufferRef pixelBuffer;
- (id)initWithFrame:(CGRect)frame;
- (void)resetRenderBuffer;
@end
#import "AAPLEAGLLayer.h"
#import
#import
#include
#import
#include
#include
#include
// Uniform index.
enum
{
UNIFORM_Y,
UNIFORM_UV,
UNIFORM_ROTATION_ANGLE,
UNIFORM_COLOR_CONVERSION_MATRIX,
NUM_UNIFORMS
};
GLint uniforms[NUM_UNIFORMS];
// Attribute index.
enum
{
ATTRIB_VERTEX,
ATTRIB_TEXCOORD,
NUM_ATTRIBUTES
};
// Color Conversion Constants (YUV to RGB) including adjustment from 16-235/16-240 (video range)
// BT.601, which is the standard for SDTV.
static const GLfloat kColorConversion601[] = {
1.164, 1.164, 1.164,
0.0, -0.392, 2.017,
1.596, -0.813, 0.0,
};
// BT.709, which is the standard for HDTV.
static const GLfloat kColorConversion709[] = {
1.164, 1.164, 1.164,
0.0, -0.213, 2.112,
1.793, -0.533, 0.0,
};
@interface AAPLEAGLLayer (){
// The pixel dimensions of the CAEAGLLayer.
GLint _backingWidth;
GLint _backingHeight;
EAGLContext *_context;
CVOpenGLESTextureRef _lumaTexture;
CVOpenGLESTextureRef _chromaTexture;
GLuint _frameBufferHandle;
GLuint _colorBufferHandle;
const GLfloat *_preferredConversion;
}
@property GLuint program;
@end
@implementation AAPLEAGLLayer
@synthesize pixelBuffer = _pixelBuffer;
-(CVPixelBufferRef) pixelBuffer{
return _pixelBuffer;
}
- (void)setPixelBuffer:(CVPixelBufferRef)pb{
if(_pixelBuffer) {
CVPixelBufferRelease(_pixelBuffer);
}
_pixelBuffer = CVPixelBufferRetain(pb);
int frameWidth = (int)CVPixelBufferGetWidth(_pixelBuffer);
int frameHeight = (int)CVPixelBufferGetHeight(_pixelBuffer);
[self displayPixelBuffer:_pixelBuffer width:frameWidth height:frameHeight];
}
- (instancetype)initWithFrame:(CGRect)frame{
self = [super init];
if (self) {
CGFloat scale = [[UIScreen mainScreen] scale];
self.contentsScale = scale;
self.opaque = TRUE;
self.drawableProperties = @{ kEAGLDrawablePropertyRetainedBacking :[NSNumber numberWithBool:YES]};
[self setFrame:frame];
// Set the context into which the frames will be drawn.
_context = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
if (!_context) {
return nil;
}
// Set the default conversion to BT.709, which is the standard for HDTV.
_preferredConversion = kColorConversion709;
[self setupGL];
}
return self;
}
/// 显示，比较复杂
- (void)displayPixelBuffer:(CVPixelBufferRef)pixelBuffer width:(uint32_t)frameWidth height:(uint32_t)frameHeight{
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
/// 显示数据为空
if(pixelBuffer == NULL) {
NSLog(@"Pixel buffer is null");
return;
}
CVReturn err;
/// 获取像素缓存区planeCount
size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
/*
Use the color attachment of the pixel buffer to determine the appropriate color conversion matrix.
参数1， 像素缓存区
参数2. YUV->RGB.kCVImageBufferYCbCrMatrixKey
参数3. 附件模式NULL
*/
CFTypeRef colorAttachments = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
if (CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) {
_preferredConversion = kColorConversion601;
}
else {
_preferredConversion = kColorConversion709;
}
/*
CVOpenGLESTextureCacheCreateTextureFromImage will create GLES texture optimally from CVPixelBufferRef.
*/
/*
Create Y and UV textures from the pixel buffer. These textures will be drawn on the frame buffer Y-plane.
*/
/// Y纹理/UV纹理！ 天然而成！-->纹理不是说已经准备好了，自己需要根据数据区创建纹理
CVOpenGLESTextureCacheRef _videoTextureCache;
// Create CVOpenGLESTextureCacheRef for optimal CVPixelBufferRef to GLES texture conversion.
err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _context, NULL, &_videoTextureCache);
if (err != noErr) {
NSLog(@"Error at CVOpenGLESTextureCacheCreate %d", err);
return;
}
/// Y纹理/UV纹理
/// 激活纹理
glActiveTexture(GL_TEXTURE0);
///1 。 创建Y纹理
err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RED_EXT,
frameWidth,
frameHeight,
GL_RED_EXT,
GL_UNSIGNED_BYTE,
0,
&_lumaTexture);
if (err) {
NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err);
}
/// 绑定纹理
glBindTexture(CVOpenGLESTextureGetTarget(_lumaTexture), CVOpenGLESTextureGetName(_lumaTexture));
///2. 设置纹理的属性
/// 放大缩小的过滤
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
/// 环绕方式S/T
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
/// 亮度Y纹理！ 已经可以成像(黑白效果！)
/// UV纹理
if(planeCount == 2) {
// UV-plane.
/// 激活纹理
glActiveTexture(GL_TEXTURE1);
/// 创建Y纹理
err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RG_EXT,
frameWidth / 2,
frameHeight / 2,
GL_RG_EXT,
GL_UNSIGNED_BYTE,
1,
&_chromaTexture);
if (err) {
NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err);
}
/// 绑定纹理
glBindTexture(CVOpenGLESTextureGetTarget(_chromaTexture), CVOpenGLESTextureGetName(_chromaTexture));
///2. 设置纹理的属性
/// 放大缩小的过滤
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
/// 环绕方式S/T
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
// Set the view port to the entire view.
glViewport(0, 0, _backingWidth, _backingHeight);
/// 清理屏幕
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
// Use shader program.
/// 使用program
glUseProgram(self.program);
// glUniform1f(uniforms[UNIFORM_LUMA_THRESHOLD], 1);
// glUniform1f(uniforms[UNIFORM_CHROMA_THRESHOLD], 1);
/// 传递值--渲染角度
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
// Set up the quad vertices with respect to the orientation and aspect ratio of the video.
/// 根据视频方向和纵横比来设置最终显示的视频frame
CGRect viewBounds = self.bounds;
CGSize contentSize = CGSizeMake(frameWidth, frameHeight);
/// 计算纵横比
/// 参数1: 纵横比
/// 参数2: 填充rect
CGRect vertexSamplingRect = AVMakeRectWithAspectRatioInsideRect(contentSize, viewBounds);
// Compute normalized quad coordinates to draw the frame into.
CGSize normalizedSamplingSize = CGSizeMake(0.0, 0.0);
CGSize cropScaleAmount = CGSizeMake(vertexSamplingRect.size.width/viewBounds.size.width,
vertexSamplingRect.size.height/viewBounds.size.height);
// Normalize the quad vertices.
if (cropScaleAmount.width > cropScaleAmount.height) {
normalizedSamplingSize.width = 1.0;
normalizedSamplingSize.height = cropScaleAmount.height/cropScaleAmount.width;
}
else {
normalizedSamplingSize.width = cropScaleAmount.width/cropScaleAmount.height;
normalizedSamplingSize.height = 1.0;;
}
/*
The quad vertex data defines the region of 2D plane onto which we draw our pixel buffers.
Vertex data formed using (-1,-1) and (1,1) as the bottom left and top right coordinates respectively, covers the entire screen.
*/
/// OpenGL ES （-1，1）
GLfloat quadVertexData [] = {
-1 * normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
-1 * normalizedSamplingSize.width, normalizedSamplingSize.height,
normalizedSamplingSize.width, normalizedSamplingSize.height,
};
// Update attribute values.
/// 坐标值- >vsh
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, quadVertexData);
glEnableVertexAttribArray(ATTRIB_VERTEX);
/*
The texture vertices are set up such that we flip the texture vertically. This is so that our top left origin buffers match OpenGL's bottom left texture coordinate system.
*/
/// 纹理坐标
CGRect textureSamplingRect = CGRectMake(0, 0, 1, 1);
GLfloat quadTextureData[] = {
CGRectGetMinX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMinX(textureSamplingRect), CGRectGetMinY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMinY(textureSamplingRect)
};
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, 0, 0, quadTextureData);
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
/// 绘制数据
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// 从渲染缓存区->屏幕上
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
[_context presentRenderbuffer:GL_RENDERBUFFER];
/// 已经完成了显示动作
/// 显示下一帧数据
[self cleanUpTextures];
// Periodic texture cache flush every frame
CVOpenGLESTextureCacheFlush(_videoTextureCache, 0);
if(_videoTextureCache) {
CFRelease(_videoTextureCache);
}
}
# pragma mark - OpenGL setup
- (void)setupGL{
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
[self setupBuffers];
[self loadShaders];
glUseProgram(self.program);
// 0 and 1 are the texture IDs of _lumaTexture and _chromaTexture respectively.
glUniform1i(uniforms[UNIFORM_Y], 0);
glUniform1i(uniforms[UNIFORM_UV], 1);
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
}
#pragma mark - Utilities
- (void)setupBuffers{
/// 取消深度测试-- >深度问题，一个时间节点下只显示一张图片，平面
glDisable(GL_DEPTH_TEST);
/// 配置定点信息
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
/// 配置纹理坐标信息（x,y）
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
/// 创建buffer
[self createBuffers];
}
- (void)createBuffers{
/// 创建帧缓存区
glGenFramebuffers(1, &_frameBufferHandle);
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
/// 创建渲染缓存区
glGenRenderbuffers(1, &_colorBufferHandle);
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
[_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:self];
/// 设置渲染缓存区的高/宽
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &_backingWidth);
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &_backingHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _colorBufferHandle);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
}
}
- (void)releaseBuffers{
if(_frameBufferHandle) {
glDeleteFramebuffers(1, &_frameBufferHandle);
_frameBufferHandle = 0;
}
if(_colorBufferHandle) {
glDeleteRenderbuffers(1, &_colorBufferHandle);
_colorBufferHandle = 0;
}
}
- (void)resetRenderBuffer{
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
[self releaseBuffers];
[self createBuffers];
}
- (void)cleanUpTextures
{
if (_lumaTexture) {
CFRelease(_lumaTexture);
_lumaTexture = NULL;
}
if (_chromaTexture) {
CFRelease(_chromaTexture);
_chromaTexture = NULL;
}
}
#pragma mark - OpenGL ES 2 shader compilation
const GLchar *shader_fsh = (const GLchar*)"varying highp vec2 texCoordVarying;"
"precision mediump float;"
"uniform sampler2D SamplerY;"
"uniform sampler2D SamplerUV;"
"uniform mat3 colorConversionMatrix;"
"void main()"
"{"
" mediump vec3 yuv;"
" lowp vec3 rgb;"
// Subtract constants to map the video range start at 0
" yuv.x = (texture2D(SamplerY, texCoordVarying).r - (16.0/255.0));"
" yuv.yz = (texture2D(SamplerUV, texCoordVarying).rg - vec2(0.5, 0.5));"
" rgb = colorConversionMatrix * yuv;"
" gl_FragColor = vec4(rgb, 1);"
"}";
const GLchar *shader_vsh = (const GLchar*)"attribute vec4 position;"
"attribute vec2 texCoord;"
"uniform float preferredRotation;"
"varying vec2 texCoordVarying;"
"void main()"
"{"
" mat4 rotationMatrix = mat4(cos(preferredRotation), -sin(preferredRotation), 0.0, 0.0,"
" sin(preferredRotation), cos(preferredRotation), 0.0, 0.0,"
" 0.0, 0.0, 1.0, 0.0,"
" 0.0, 0.0, 0.0, 1.0);"
" gl_Position = position * rotationMatrix;"
" texCoordVarying = texCoord;"
"}";
- (BOOL)loadShaders{
GLuint vertShader = 0, fragShader = 0;
// Create the shader program.
self.program = glCreateProgram();
if(![self compileShaderString:&vertShader type:GL_VERTEX_SHADER shaderString:shader_vsh]) {
NSLog(@"Failed to compile vertex shader");
return NO;
}
if(![self compileShaderString:&fragShader type:GL_FRAGMENT_SHADER shaderString:shader_fsh]) {
NSLog(@"Failed to compile fragment shader");
return NO;
}
// Attach vertex shader to program.
glAttachShader(self.program, vertShader);
// Attach fragment shader to program.
glAttachShader(self.program, fragShader);
// Bind attribute locations. This needs to be done prior to linking.
glBindAttribLocation(self.program, ATTRIB_VERTEX, "position");
glBindAttribLocation(self.program, ATTRIB_TEXCOORD, "texCoord");
// Link the program.
if (![self linkProgram:self.program]) {
NSLog(@"Failed to link program: %d", self.program);
if (vertShader) {
glDeleteShader(vertShader);
vertShader = 0;
}
if (fragShader) {
glDeleteShader(fragShader);
fragShader = 0;
}
if (self.program) {
glDeleteProgram(self.program);
self.program = 0;
}
return NO;
}
// Get uniform locations.
uniforms[UNIFORM_Y] = glGetUniformLocation(self.program, "SamplerY");
uniforms[UNIFORM_UV] = glGetUniformLocation(self.program, "SamplerUV");
// uniforms[UNIFORM_LUMA_THRESHOLD] = glGetUniformLocation(self.program, "lumaThreshold");
// uniforms[UNIFORM_CHROMA_THRESHOLD] = glGetUniformLocation(self.program, "chromaThreshold");
uniforms[UNIFORM_ROTATION_ANGLE] = glGetUniformLocation(self.program, "preferredRotation");
uniforms[UNIFORM_COLOR_CONVERSION_MATRIX] = glGetUniformLocation(self.program, "colorConversionMatrix");
// Release vertex and fragment shaders.
if (vertShader) {
glDetachShader(self.program, vertShader);
glDeleteShader(vertShader);
}
if (fragShader) {
glDetachShader(self.program, fragShader);
glDeleteShader(fragShader);
}
return YES;
}
- (BOOL)compileShaderString:(GLuint *)shader type:(GLenum)type shaderString:(const GLchar*)shaderString
{
*shader = glCreateShader(type);
glShaderSource(*shader, 1, &shaderString, NULL);
glCompileShader(*shader);
#if defined(DEBUG)
GLint logLength;
glGetShaderiv(*shader, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetShaderInfoLog(*shader, logLength, &logLength, log);
NSLog(@"Shader compile log:\n%s", log);
free(log);
}
#endif
GLint status = 0;
glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
if (status == 0) {
glDeleteShader(*shader);
return NO;
}
return YES;
}
- (BOOL)compileShader:(GLuint *)shader type:(GLenum)type URL:(NSURL *)URL
{
NSError *error;
NSString *sourceString = [[NSString alloc] initWithContentsOfURL:URL encoding:NSUTF8StringEncoding error:&error];
if (sourceString == nil) {
NSLog(@"Failed to load vertex shader: %@", [error localizedDescription]);
return NO;
}
const GLchar *source = (GLchar *)[sourceString UTF8String];
return [self compileShaderString:shader type:type shaderString:source];
}
- (BOOL)linkProgram:(GLuint)prog
{
GLint status;
glLinkProgram(prog);
#if defined(DEBUG)
GLint logLength;
glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetProgramInfoLog(prog, logLength, &logLength, log);
NSLog(@"Program link log:\n%s", log);
free(log);
}
#endif
glGetProgramiv(prog, GL_LINK_STATUS, &status);
if (status == 0) {
return NO;
}
return YES;
}
- (BOOL)validateProgram:(GLuint)prog
{
GLint logLength, status;
glValidateProgram(prog);
glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetProgramInfoLog(prog, logLength, &logLength, log);
NSLog(@"Program validate log:\n%s", log);
free(log);
}
glGetProgramiv(prog, GL_VALIDATE_STATUS, &status);
if (status == 0) {
return NO;
}
return YES;
}
- (void)dealloc
{
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
[self cleanUpTextures];
if(_pixelBuffer) {
CVPixelBufferRelease(_pixelBuffer);
}
if (self.program) {
glDeleteProgram(self.program);
self.program = 0;
}
if(_context) {
//[_context release];
_context = nil;
}
//[super dealloc];
}