OpenGL.Shader：志哥教你写一个滤镜直播客户端（3）OpenGL渲染NV21(YUV)
OpenGL.Shader：志哥教你写一个滤镜直播客户端（3）
上一章说到两个点，第一个是帧数据缓存池，缓存NV21帧数据（调用栈是JClass Activity->JClass CFEScheduler->JCallback Camera.onPreviewFrame->JNIMethod feedVideoData）；第二个是setRotationCamera，根据旋转角度，调整水平翻转和垂直翻转（调用栈是JClass Activity->JClass CFEScheduler->JMethod setUpCamera->JNIMethod setRotationCamera）；两者最终都会触发adjustFrameScaling，调整帧图缩放比例，生成用于OpenGL渲染的顶点坐标数据和纹理坐标数据。
接下来开始真正的把图像数据渲染到屏幕载体上。这次和前篇介绍的水印录制有些不一样，前篇介绍的方法是通过系统准备好的纹理对象，我们借用其纹理ID直接操作就完事了，。但这一次只有最原始的NV21格式的图像帧数据，如何高效的把图像帧数据渲染到输出载体surface成为主要的问题关键。
还要一个问题需要注意，往后添加各种滤镜效果，如何做到无缝切换？（暂时先放一个point在这）事不宜迟，show code.
首先看一下GLThread+GLRender的三大生命周期回调的内容：（如有疑问请参考以前的文章https://blog.csdn.net/a360940265a/article/details/88600962）
void GpuFilterRender::surfaceCreated(ANativeWindow *window)
{
if (mEglCore == NULL) {
mEglCore = new EglCore(NULL, FLAG_TRY_GLES2);
}
mWindowSurface = new WindowSurface(mEglCore, window, true);
assert(mWindowSurface != NULL && mEglCore != NULL);
LOGD("render surface create ... ");
mWindowSurface->makeCurrent();
if( mFilter==NULL) {
mFilter = new GpuBaseFilter();
} else {
mFilter->destroy();
}
mFilter->init();
mWindowSurface->swapBuffers();
// ...
}
void GpuFilterRender::surfaceChanged(int width, int height) {
this->mViewWidth = width;
this->mViewHeight = height;
mWindowSurface->makeCurrent();
mFilter->onOutputSizeChanged(width, height);
mWindowSurface->swapBuffers();
}
void GpuFilterRender::surfaceDestroyed() {
if (mWindowSurface) {
mWindowSurface->release();
delete mWindowSurface;
mWindowSurface = NULL;
}
if (mEglCore) {
mEglCore->release();
delete mEglCore;
mEglCore = NULL;
}
}显然这个GpuBaseFilter将会是一个关键类，在整个滤镜渲染的过程中起到一个至关重要的作用。
所以赶紧看看这个GpuBaseFilter做了些什么。
#ifndef GPU_NORMAL_FILTER_HPP
#define GPU_NORMAL_FILTER_HPP
#define FILTER_TYPE_NORMAL 0x1010
/**
* Filter基础类，支持YUV / RGB渲染模式。
*/
class GpuBaseFilter {
public:
// 用于上层获取滤镜列表对应的Filter类型
virtual int getTypeId() { return FILTER_TYPE_NORMAL; }
GpuBaseFilter()
{
NO_FILTER_VERTEX_SHADER = "attribute vec4 position;\n\
attribute vec4 inputTextureCoordinate;\n\
varying vec2 textureCoordinate;\n\
void main()\n\
{\n\
gl_Position = position;\n\
textureCoordinate = inputTextureCoordinate.xy;\n\
}";
NO_FILTER_FRAGMENT_SHADER = "precision mediump float;\n\
varying highp vec2 textureCoordinate;\n\
uniform sampler2D SamplerRGB;\n\
uniform sampler2D SamplerY;\n\
uniform sampler2D SamplerU;\n\
uniform sampler2D SamplerV;\n\
mat3 colorConversionMatrix = mat3(\n\
1.0, 1.0, 1.0,\n\
0.0, -0.39465, 2.03211,\n\
1.13983, -0.58060, 0.0);\n\
vec3 yuv2rgb(vec2 pos)\n\
{\n\
vec3 yuv;\n\
yuv.x = texture2D(SamplerY, pos).r;\n\
yuv.y = texture2D(SamplerU, pos).r - 0.5;\n\
yuv.z = texture2D(SamplerV, pos).r - 0.5;\n\
return colorConversionMatrix * yuv;\n\
}\n\
void main()\n\
{\n\
gl_FragColor = vec4(yuv2rgb(textureCoordinate), 1.0);\n\
//gl_FragColor = texture2D(SamplerY/U/V, textureCoordinate);\n\
}";
}
virtual ~GpuBaseFilter()
{
if(!NO_FILTER_VERTEX_SHADER.empty()) NO_FILTER_VERTEX_SHADER.clear();
if(!NO_FILTER_FRAGMENT_SHADER.empty()) NO_FILTER_FRAGMENT_SHADER.clear();
mIsInitialized = false;
}
virtual void init() {
init(NO_FILTER_VERTEX_SHADER.c_str(), NO_FILTER_FRAGMENT_SHADER.c_str());
}
void init(const char *vertexShaderSource, const char *fragmentShaderSource) {
mGLProgId = ShaderHelper::buildProgram(vertexShaderSource, fragmentShaderSource);
mGLAttribPosition = static_cast(glGetAttribLocation(mGLProgId, "position"));
mGLUniformSampleRGB = static_cast(glGetUniformLocation(mGLProgId, "SamplerRGB"));
mGLUniformSampleY = static_cast(glGetUniformLocation(mGLProgId, "SamplerY"));
mGLUniformSampleU = static_cast(glGetUniformLocation(mGLProgId, "SamplerU"));
mGLUniformSampleV = static_cast(glGetUniformLocation(mGLProgId, "SamplerV"));
mGLAttribTextureCoordinate = static_cast(glGetAttribLocation(mGLProgId, "inputTextureCoordinate"));
mIsInitialized = true;
}
virtual void destroy() {
mIsInitialized = false;
glDeleteProgram(mGLProgId);
}
virtual void onOutputSizeChanged(int width, int height) {
mOutputWidth = width;
mOutputHeight = height;
}
virtual void onDraw(GLuint SamplerY_texId, GLuint SamplerU_texId, GLuint SamplerV_texId,
void* positionCords, void* textureCords)
{
if (!mIsInitialized)
return;
glUseProgram(mGLProgId);
// runPendingOnDrawTasks();
glVertexAttribPointer(mGLAttribPosition, 2, GL_FLOAT, GL_FALSE, 0, positionCords);
glEnableVertexAttribArray(mGLAttribPosition);
glVertexAttribPointer(mGLAttribTextureCoordinate, 2, GL_FLOAT, GL_FALSE, 0, textureCords);
glEnableVertexAttribArray(mGLAttribTextureCoordinate);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, SamplerY_texId);
glUniform1i(mGLUniformSampleY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, SamplerU_texId);
glUniform1i(mGLUniformSampleU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, SamplerV_texId);
glUniform1i(mGLUniformSampleV, 2);
// onDrawArraysPre();
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glDisableVertexAttribArray(mGLAttribPosition);
glDisableVertexAttribArray(mGLAttribTextureCoordinate);
glBindTexture(GL_TEXTURE_2D, 0);
}
// GLUniformSampleRGB对应的draw，我这里没用rgb模式.
//virtual void onDraw2(GLuint textureId, void* positionCords, void* textureCords)
//{
// if (!mIsInitialized)
// return;
// glUseProgram(mGLProgId);
// // runPendingOnDrawTasks();
// glVertexAttribPointer(mGLAttribPosition, 2, GL_FLOAT, GL_FALSE, 0, positionCords);
// glEnableVertexAttribArray(mGLAttribPosition);
// glVertexAttribPointer(mGLAttribTextureCoordinate, 2, GL_FLOAT, GL_FALSE, 0, textureCords);
// glEnableVertexAttribArray(mGLAttribTextureCoordinate);
// if (textureId != -1) {
// glActiveTexture(GL_TEXTURE0);
// glBindTexture(GL_TEXTURE_2D, textureId);
// glUniform1i(mGLUniformSampleRGB, 0);
// }
// // onDrawArraysPre();
// glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// glDisableVertexAttribArray(mGLAttribPosition);
// glDisableVertexAttribArray(mGLAttribTextureCoordinate);
// glBindTexture(GL_TEXTURE_2D, 0);
//}
// 相关滤镜对应的可调整参数，通过此借口进行操作
virtual void setAdjustEffect(float percent) {
// subclass override
}
bool isInitialized(){ return mIsInitialized;}
GLuint getProgram(){ return mGLProgId;}
protected:
std::string NO_FILTER_VERTEX_SHADER;
std::string NO_FILTER_FRAGMENT_SHADER;
GLuint mGLProgId;
GLuint mGLAttribPosition;
GLuint mGLUniformSampleRGB;
GLuint mGLAttribTextureCoordinate;
GLuint mGLUniformSampleY;
GLuint mGLUniformSampleU;
GLuint mGLUniformSampleV;
int mOutputWidth;
int mOutputHeight;
bool mIsInitialized;
};
#endif // GPU_NORMAL_FILTER_HPP 篇幅有点长，这是一个标准C++创建的基类，带virtual关键字开头的函数都将被继承类所改写（相关知识点请自行百度）重点关注关于Shader的构造函数 和 关于渲染的onDraw方法。
先看看顶点着色器 NO_FILTER_VERTEX_SHADER：
attribute vec4 position;
attribute vec4 inputTextureCoordinate;
varying vec2 textureCoordinate;
void main()
{
gl_Position = position;
textureCoordinate = inputTextureCoordinate.xy;
}
内容不复杂，循规蹈矩的输出顶点坐标，把纹理坐标传递到片元着色器。
接下来再看 NO_FILTER_FRAGMENT_SHADER，这是本篇内容的重点之一
precision mediump float;
varying highp vec2 textureCoordinate;
uniform sampler2D SamplerRGB;
uniform sampler2D SamplerY;
uniform sampler2D SamplerU;
uniform sampler2D SamplerV;
mat3 colorConversionMatrix = mat3(
1.0, 1.0, 1.0,
0.0, -0.39465, 2.03211,
1.13983, -0.58060, 0.0);
vec3 yuv2rgb(vec2 pos)
{
vec3 yuv;
yuv.x = texture2D(SamplerY, pos).r;
yuv.y = texture2D(SamplerU, pos).r - 0.5;
yuv.z = texture2D(SamplerV, pos).r - 0.5;
return colorConversionMatrix * yuv;
}
void main()
{
gl_FragColor = vec4(yuv2rgb(textureCoordinate), 1.0);
//gl_FragColor = texture2D(SamplerRGB, textureCoordinate);
}
显然这里做了两手的准备，SamplerRGB是彩色纹理，用于注释掉的那句代码，根据纹理坐标直接得出gl_FragColor；SamplerY/U/V看名字可以知道三者是分别储存yuv格式的三种分量，然后通过yuv转rgb的通用公式：
R = Y + 0*U + 1.140*V
G = Y - 0.395*U - 0.581*V
B = Y + 2.032*U + 0*V
以上公式是未量化的小数形式。何为量化？
量化就是让通过线性变换让Y或U或V处于一定的范围内， 假设量化前Y/U/V~(0-255)，量化后参数: Y~(16,235) U ~(16-240) V~(16-240)，让Y(0,255) 变到 Y' (16,235) 就这样来实行：Y' = Y*[(235-16)/255] +16。
还有一点就是，OpenGL的矩阵是列优先，所以在自定义矩阵的时候，需要注意行列互换。
Shader准备完毕之后，接下来就是要如何针对NV21进行渲染了。
void GpuFilterRender::renderOnDraw(double elpasedInMilliSec)
{
if (mEglCore == NULL || mWindowSurface == NULL) {
LOGW("Skipping drawFrame after shutdown");
return;
}
pthread_mutex_lock(&mutex);
ByteBuffer* item = mNV21Pool.get();
if(item == NULL) {
pthread_mutex_unlock(&mutex);
return;
} else { // item!=NULL，i420BufferY/U/V也!=NULL
int8_t * nv21_buffer = item->data();
int y_len = item->param1;
int u_len = item->param2;
int v_len = item->param3;
// 装填y u v数据。
int8_t * dst_y = i420BufferY->data();
int8_t * dst_u = i420BufferU->data();
int8_t * dst_v = i420BufferV->data();
memcpy(dst_y, nv21_buffer, (size_t) y_len);
for (int i = 0; i < u_len; i++) {
//NV21 先v后u
*(dst_v + i) = (uint8_t) *(nv21_buffer + y_len + i * 2);
*(dst_u + i) = (uint8_t) *(nv21_buffer + y_len + i * 2 + 1);
}
// 删除BufferPool当中的引用。
delete item;
pthread_mutex_unlock(&mutex);//#赶紧解锁
// 画面渲染
mWindowSurface->makeCurrent();
yTextureId = updateTexture(dst_y, yTextureId, mFrameWidth, mFrameHeight);
uTextureId = updateTexture(dst_u, uTextureId, mFrameWidth/2, mFrameHeight/2);
vTextureId = updateTexture(dst_v, vTextureId, mFrameWidth/2, mFrameHeight/2);
if( mFilter!=NULL) {
mFilter->onDraw(yTextureId, uTextureId, vTextureId, positionCords, textureCords);
}
mWindowSurface->swapBuffers();
}
}
GLuint GpuFilterRender::updateTexture(int8_t *src, int texId, int width, int height)
{
GLuint mTextureID;
if( texId == -1) {
glGenTextures(1, &mTextureID);
glBindTexture(GL_TEXTURE_2D, mTextureID);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height,
0, GL_LUMINANCE, GL_UNSIGNED_BYTE, src);
} else {
glBindTexture(GL_TEXTURE_2D, texId);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
GL_LUMINANCE, GL_UNSIGNED_BYTE, src);
mTextureID = texId;
}
glBindTexture(GL_TEXTURE_2D, 0);
return mTextureID;
}回顾一下我们在feedVideoData时候填充nv21数据到缓存池mNV21Pool，在GLThread->renderOnDraw的回调当中，我们就不停的获取nv21，然后按照NV21的格式（YYYYVU ...），分解成Y/U/V三个buffer，再然后通过updateTexture方法，及时更新纹理对象。这里有个容易出错的地方需要注意，Ybuffer的长宽是正常的FrameSize，但是UV的长宽是FrameSize/2，如果没有正确的对应长宽，glTexImage2D / glTexSubImage2D会报SIGSEGV的错误崩溃。往后就可以通过GpuBaseFilter.onDraw渲染了。
virtual void onDraw(GLuint SamplerY_texId, GLuint SamplerU_texId, GLuint SamplerV_texId,
void* positionCords, void* textureCords)
{
if (!mIsInitialized)
return;
glUseProgram(mGLProgId);
//绑定顶点坐标 和 纹理坐标
glVertexAttribPointer(mGLAttribPosition, 2, GL_FLOAT, GL_FALSE, 0, positionCords);
glEnableVertexAttribArray(mGLAttribPosition);
glVertexAttribPointer(mGLAttribTextureCoordinate, 2, GL_FLOAT, GL_FALSE, 0, textureCords);
glEnableVertexAttribArray(mGLAttribTextureCoordinate);
// 把Y/U/V分别绑定三个纹理对象
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, SamplerY_texId);
glUniform1i(mGLUniformSampleY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, SamplerU_texId);
glUniform1i(mGLUniformSampleU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, SamplerV_texId);
glUniform1i(mGLUniformSampleV, 2);
// 经过shader的渲染
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glDisableVertexAttribArray(mGLAttribPosition);
glDisableVertexAttribArray(mGLAttribTextureCoordinate);
glBindTexture(GL_TEXTURE_2D, 0);
}
整个流程到此走通，到这里我们已经可以看到适配屏幕比例的摄像头视频源，下一章主要内容是通过设置三个简单的滤镜效果，提供一个无缝切换滤镜的通用方法。
项目地址：https://github.com/MrZhaozhirong/NativeCppApp