相机之使用OpenGL预览
界面全屏
在MainActivity中设置界面全屏
class MainActivity : AppCompatActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
//无标题、全屏
supportRequestWindowFeature(Window.FEATURE_NO_TITLE)
window.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN)
//使用了navigation的框架,设置layout后,会根据nav_graph自动转到startDestination属性设置的fragment中,即PermissionsFragment
setContentView(R.layout.activity_main)
}
}
navigation框架
采用navigation的框架,一共三个Fragment,如图所示
SelectorAdapter
分辨率列表适配器
class SelectorAdapter(private val clickListener: OptionClickListener) :
RecyclerView.Adapter() {
//item的数据
var data = listOf()
set(value) {
field = value
notifyDataSetChanged()
}
override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder {
//在ViewHolder中创建各个item
return ViewHolder.from(parent)
}
override fun getItemCount(): Int {
return data.size
}
override fun onBindViewHolder(holder: ViewHolder, position: Int) {
//在ViewHolder中绑定各个item
holder.bind(data, position, clickListener)
}
class ViewHolder constructor(private val binding: SelectorItemBinding) :
RecyclerView.ViewHolder(binding.root) {
/**
* 绑定数据和设置监听
*/
fun bind(data: List, position: Int, clickListener: OptionClickListener) {
binding.optionItem = data[position]
//为每个item绑定点击监听
binding.clickListener = clickListener
binding.resolution = data[position].formatOption()
//每次绑定更新可能导致View更改其大小,如果在下一帧中推迟计算可能导致测量读取错误的值
//因此调用executePendingBindings强制绑定以同步方式更新数据
binding.executePendingBindings()
}
companion object {
/**
* 得到一个ViewHolder
*/
fun from(parent: ViewGroup): ViewHolder {
var inflater = LayoutInflater.from(parent.context)
//得到通过item布局的binding,下面两种方式都行
/*var binding =
DataBindingUtil.inflate(
inflater,
R.layout.selector_item,
parent,
false
)*/
var binding = SelectorItemBinding.inflate(inflater, parent, false)
//创建并返回ViewHolder
return ViewHolder(binding)
}
}
}
}
/**
* item点击的监听,用于外部设置
* 这里是在xml中声明对象,创建是在SelectorFragment中,然后传入本适配器,在绑定的时候为xml中声明的对象赋值,点击item的时候调用
*/
class OptionClickListener(val clickListener: (OptionItem) -> Unit) {
fun onClick(optionItem: OptionItem) = clickListener(optionItem)
}
ViewModel
class SelectorViewModel : ViewModel() {
//一个LiveData,可观察对象
val checkedOptionItem = MutableLiveData()
/**
* 改变选择的分辨率选项,checkedOptionItem对象的观察者都会执行onChanged方法
*/
fun onOptionChanged(optionItem: OptionItem) {
checkedOptionItem.value = optionItem
}
}
相机预览
主要流程
- 打开摄像头
- 将摄像头数据绘制到 FBO 中,因为可以做一些对图像的处理操作
- 将 FBO 中的数据绘制到屏幕上
使用CameraUtil打开摄像头
调用CameraUtil.startPreview(cameraId: String, outputs: List
class CameraUtil {
private lateinit var session: CameraCaptureSession
private lateinit var cameraDevice: CameraDevice
private lateinit var outputs: List
private val cameraThread = HandlerThread("cameraThread").apply { start() }
private val cameraHandler = Handler(cameraThread.looper)
private val cameraManager: CameraManager
private val context: Context
companion object {
private const val TAG = "CameraUtil"
}
constructor(context: Context) {
this.context = context
cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
}
/**
* 开始预览
*/
suspend fun startPreview(
cameraId: String,
outputs: List
) {
this.outputs = outputs
cameraDevice = openCamera(cameraId)
session = createCaptureSession(cameraDevice, outputs)
var request =
cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply {
outputs.forEach {
addTarget(it)
}
}
session.setRepeatingRequest(
request.build(),
object : CameraCaptureSession.CaptureCallback() {
override fun onCaptureCompleted(
session: CameraCaptureSession?,
request: CaptureRequest?,
result: TotalCaptureResult?
) {
super.onCaptureCompleted(session, request, result)
// Log.d(TAG, "onCaptureCompleted: ")
}
override fun onCaptureFailed(
session: CameraCaptureSession?,
request: CaptureRequest?,
failure: CaptureFailure?
) {
super.onCaptureFailed(session, request, failure)
// Log.d(TAG, "onCaptureFailed: ")
}
override fun onCaptureStarted(
session: CameraCaptureSession?,
request: CaptureRequest?,
timestamp: Long,
frameNumber: Long
) {
super.onCaptureStarted(session, request, timestamp, frameNumber)
// Log.d(TAG, "onCaptureStarted: ")
}
},
cameraHandler
)
}
/**
* 创建一个CameraCaptureSession
*/
private suspend fun createCaptureSession(
cameraDevice: CameraDevice,
outputs: List
): CameraCaptureSession =
suspendCancellableCoroutine {
cameraDevice.createCaptureSession(
outputs,
object : CameraCaptureSession.StateCallback() {
override fun onConfigureFailed(session: CameraCaptureSession) {
Log.e(TAG, "createCaptureSession onConfigureFailed: ")
it.cancel()
}
override fun onConfigured(session: CameraCaptureSession) {
it.resume(session)
}
}, cameraHandler
)
}
/**
* 打开摄像头,获取CameraDevice
*/
private suspend fun openCamera(cameraId: String): CameraDevice =
suspendCancellableCoroutine { cont ->
if (ActivityCompat.checkSelfPermission(
context,
Manifest.permission.CAMERA
) != PackageManager.PERMISSION_GRANTED
) {
Log.e(TAG, "openCamera: has not permission!")
cont.cancel()
}
cameraManager.openCamera(cameraId, object : CameraDevice.StateCallback() {
override fun onOpened(camera: CameraDevice) {
cont.resume(camera)
}
override fun onDisconnected(camera: CameraDevice?) {
Log.e(TAG, "onDisconnected Camera $cameraId has been disconnected")
cont.cancel()
}
override fun onError(camera: CameraDevice?, error: Int) {
var msg = when (error) {
ERROR_CAMERA_DEVICE -> "fatal (device)"
ERROR_CAMERA_DISABLED -> "device policy"
ERROR_CAMERA_IN_USE -> "camera in use"
ERROR_CAMERA_SERVICE -> "fatal (device)"
ERROR_MAX_CAMERAS_IN_USE -> "Maximum cameras in use"
else -> "unknown error"
}
Log.e(TAG, "onError $msg")
cont.cancel()
}
}, cameraHandler)
}
fun release() {
session.stopRepeating()
session.close()
outputs.forEach {
it.release()
}
cameraDevice.close()
cameraThread.quitSafely()
}
}
将摄像头数据绘制到 FBO 中
FBO(Frame Buffer Object)
FBO(Frame Buffer Object),也就是帧缓冲对象。一般情况下,使用OpenGL ES经过顶点着色器、片元着色器处理之后,就通过OpenGL ES使用的窗口系统提供的默认帧缓冲区,这样绘制的结果是显示到窗口 (屏幕) 上
但是对于需要复杂的渲染处理,比如通过多个滤镜处理,则应该等所有处理流程完成之后再显示到窗口上,这时就可以使用 FBO
FBO可以理解为包含了许多挂接点的一个对象,它本身并不存储图像相关的数据。在FBO中必然包含一个深度缓冲区挂接点和一个模板缓冲区挂接点,同时还包含许多颜色缓冲区挂接点(具体多少个受OpenGL实现的影响,可以通过GL_MAX_COLOR_ATTACHMENTS使用glGet查询),FBO的这些挂接点用来挂接纹理对象和渲染对象,这两类对象中才真正存储了需要被显示的数据。FBO只是提供了一种可以快速切换外部纹理对象和渲染对象挂接点的方式,对于纹理对象使用 glFramebufferTexture2D,对于渲染对象使用glFramebufferRenderbuffer
创建 FBO
/**
* FBO即frame buffer object帧缓冲对象
* 作用就是渲染到离屏的buffer中,不显示到屏幕,FBO中并没有存储图像,只有多个关联点,
* 帧缓存对象中有多个颜色关联点、一个深度关联点,和一个模板关联,可以把帧缓冲对象理解为一个插线板,
* 自己本身没有内存,但是可以连接纹理对象和渲染缓冲对象两种外设,这两种外设是有内存的来存储图像数据
*/
class FBO constructor(width: Int, height: Int) {
private val TAG = "FBO"
var textureId = 0
var framebufferId = 0
init {
//创建帧缓冲对象将要关联的纹理
var textureIdArray = IntArray(1)
GLES20.glGenTextures(textureIdArray.size, textureIdArray, 0)
textureId = textureIdArray[0]
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId)
//加载一个2D图像作为纹理对象
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_RGBA,
width,
height,
0,
GLES20.GL_RGBA,
GLES20.GL_UNSIGNED_BYTE,
null
)
//设置纹理过滤参数,缩小和放大
//GLES20.GL_LINEAR:使用纹理中坐标附近的若干个颜色,通过平均算法 进行放大
//GLES20.GL_NEAREST:使用纹理坐标最接近的一个颜色作为放大的要绘制的颜色
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_NEAREST
)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST
)
/*设置纹理环绕方式*/
//纹理坐标的st表示xy
//纹理坐标的范围是0-1,超出这一范围的坐标将被OpenGL根据GL_TEXTURE_WRAP参数的值进行处理
//GL_TEXTURE_WRAP_S, GL_TEXTURE_WRAP_T 分别对应x,y方向
//GL_REPEAT:平铺
//GL_MIRRORED_REPEAT:纹理坐标是奇数时使用镜像平铺
//GL_CLAMP_TO_EDGE:坐标超出部分被截取成0、1,边缘拉伸
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_REPEAT)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_REPEAT)
setUpFbo()
//解绑
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
}
/**
* setup FBO
*/
private fun setUpFbo() {
//创建FBO对象(帧缓冲对象)
var framebufferIdArray = IntArray(1)
GLES20.glGenFramebuffers(framebufferIdArray.size, framebufferIdArray, 0)
framebufferId = framebufferIdArray[0]
//将这个帧缓冲对象设置为当前的帧缓冲区,绑定完成以后,接下来所有的读、写操作都会影响到当前绑定的帧缓冲对象
//这个时候帧缓冲对象还没有绑定关联点
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebufferId)
//将textureId纹理附着到帧缓冲区
GLES20.glFramebufferTexture2D(
GLES20.GL_FRAMEBUFFER,
GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D,
textureId,
0
)
var state = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER)
Log.d(TAG, "setUpFbo: glCheckFramebufferStatus=$state")
//解绑帧缓冲区
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
ErrorUtil.checkGlError("setUpFbo")
}
}
着色器采样摄像头数据
顶点着色器
attribute vec4 a_Position;
uniform mat4 u_Matrix;
attribute vec4 a_TextureCoord;
varying vec2 v_TextureCoord;
void main() {
gl_Position=a_Position;
v_TextureCoord=(u_Matrix * a_TextureCoord).xy;
}
变量 | 说明 |
---|---|
a_Position | 顶点数据,范围是 [-1,1] |
u_Matrix | 顶点数据要进行的变换矩阵 |
a_TextureCoord | 纹理坐标,范围是 [0,1] |
v_TextureCoord | 传给片段着色器,在这个坐标进行数据采样,由a_TextureCoord进行矩阵变换得到 |
gl_Position | 最终的顶点位置,是OpenGL内置变量,由它觉得显示的形状 |
因为纹理坐标和OpenGL坐标不一致,因此也需要将纹理坐标传进来,以便后面的片段着色器进行采样,顶点位置决定了在哪里绘制(例如 A 点),纹理坐标决定了在 A 点绘制纹理的哪一部分
片段着色器
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES vTexture;
varying vec2 v_TextureCoord;
void main() {
gl_FragColor=texture2D(vTexture, v_TextureCoord);
}
"#extension GL_OES_EGL_image_external : require",这一句是因为要采样摄像头数据,采样器使用的samplerExternalOES,而要使用samplerExternalOES,就需要声明这一句
texture2D是采样的一个方法
第一个参数传递的是采样器,之后会在代码中传递过来,一般情况下的采样器都是sampler2D,但摄像头要特殊一点,需要使用samplerExternalOES
第二个参数是从顶点着色器中传过来的采样坐标,texture2D的返回值,就是采样到的颜色值,将其赋值给gl_FragColor,也就是确定片段最终颜色
着色器封装代码
class FboFilter : BaseFilter {
private lateinit var matrix: FloatArray
lateinit var fbo: FBO
private val U_MATRIX: String = "u_Matrix"
private val uMatrix: Int
constructor(context: Context) : super(context, R.raw.camera_vertex, R.raw.camera_frag) {
uMatrix = GLES20.glGetUniformLocation(mProgram, U_MATRIX)
ErrorUtil.checkGlError("glGetUniformLocation uMatrix")
}
/**
* 设置着色器中uniform的值
* @param matrix FloatArray
*/
fun setUniforms(matrix: FloatArray) {
this.matrix = matrix
}
/**
* 画到FBO中,并返回FBO对应textureId
*/
override fun onDrawFrame(textureId: Int): Int {
GLES20.glUseProgram(mProgram)
//绑定顶点和纹理数据
bindData()
//绑定到FBO,就是将这个帧缓冲对象设置为当前的帧缓冲区,绑定完成以后,接下来所有的读、写操作都会影响到当前绑定的帧缓冲对象
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo.framebufferId)
//将matrix传给opengl程序中的uMatrix属性
GLES20.glUniformMatrix4fv(uMatrix, 1, false, matrix, 0)
//激活纹理单元0
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
//将textureId纹理绑定到纹理单元0
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
//将纹理单元0传给vTexture,告诉vTexture采样器从纹理单元0读取数据
GLES20.glUniform1i(vTexture, 0)
//在textureId纹理上画出图像
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
//解除绑定
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0)
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
return fbo.textureId
}
}
父类BaseFilter,用于封装通用的着色器代码
abstract class BaseFilter {
companion object {
// 位置信息占据的Float位数
const val POSITION_COUNT = 2
// 着色器程序中通用的获取位置编号的常量
private const val A_POSITION: String = "a_Position"
private const val A_TEXTURECOORD: String = "a_TextureCoord"
private const val V_TEXTURE: String = "vTexture"
// 顶点位置坐标
private val VERTEX: FloatArray = floatArrayOf(
-1f, 1f,
-1f, -1f,
1f, 1f,
1f, -1f
)
// 纹理坐标
private val TEXTURE: FloatArray = floatArrayOf(
1f, 0f,
1f, 1f,
0f, 0f,
0f, 1f
)
val vertexArray = VertexArray(VERTEX)
val textureArray = VertexArray(TEXTURE)
}
private val aPosition: Int
private val aTextureCoord: Int
val vTexture: Int
var mProgram = 0
constructor(context: Context, vertexResId: Int, fragResId: Int) {
//编译顶点着色器
val vertexId: Int = ShaderUtil.compileVertexShader(context, vertexResId)
//编译片段着色器
val fragmentId: Int = ShaderUtil.compileFragmentShader(context, fragResId)
//通过顶点着色器和片段着色器创建OpenGL程序
mProgram = ProgramUtil.getProgram(vertexId, fragmentId)
ErrorUtil.checkGlError("getProgram")
//获得mProgram程序中各个变量的索引,之后会通过这些索引为这些变量赋值
aPosition = GLES20.glGetAttribLocation(mProgram, A_POSITION)
ErrorUtil.checkGlError("glGetAttribLocation aPosition")
aTextureCoord = GLES20.glGetAttribLocation(mProgram, A_TEXTURECOORD)
ErrorUtil.checkGlError("glGetAttribLocation aTextureCoord")
vTexture = GLES20.glGetUniformLocation(mProgram, V_TEXTURE)
ErrorUtil.checkGlError("glGetUniformLocation vTexture")
}
abstract fun onDrawFrame(textureId: Int): Int
/**
* 绑定顶点和纹理数据,也就是着色器中 attribute 的值
*/
fun bindData() {
vertexArray.setVertexAttribPointer(
0,
aPosition,
POSITION_COUNT,
0
)
textureArray.setVertexAttribPointer(
0,
aTextureCoord,
POSITION_COUNT,
0
)
}
}
着色器采样FBO数据
顶点着色器
attribute vec4 a_Position;
attribute vec2 a_TextureCoord;
varying vec2 v_TextureCoord;
void main() {
gl_Position=a_Position;
v_TextureCoord=a_TextureCoord;
}
和采样摄像头数据的顶点着色器相差不大,只是不再需要使用矩阵变换顶点数据了,因为这一步已经在采样摄像头数据是做过了
片段着色器
precision mediump float;
uniform sampler2D vTexture;
varying vec2 v_TextureCoord;
void main() {
gl_FragColor=texture2D(vTexture, v_TextureCoord);
}
和采样摄像头数据的片段着色器也相差无几,只是采样器换成了sampler2D
着色器封装代码
class ScreenFilter(context: Context) : BaseFilter(context, R.raw.screen_vertex, R.raw.screen_frag) {
companion object {
private const val TAG = "ScreenFilter"
}
/**
* 画到屏幕上
*/
override fun onDrawFrame(textureId: Int): Int {
GLES20.glUseProgram(mProgram)
//绑定顶点和纹理数据
bindData()
//激活纹理单元0
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
//将textureId纹理绑定到纹理单元0
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId)
//将被激活的纹理单元0传给片段着色器中的vTexture,告诉vTexture采样器从纹理单元0读取数据
GLES20.glUniform1i(vTexture, 0)
//在textureId纹理上画出图像
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
//解除绑定
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
return textureId
}
}
在渲染器中组合代码
class GlRenderer : GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
companion object {
private const val TAG = "MyRenderer"
}
private val glSurfaceView: GLSurfaceView
private val context: Context
//用于控制摄像头,打开摄像头之类的
private var cameraUtil: CameraUtil
//将摄像头数据画到FBO中
private lateinit var fboFilter: FboFilter
//蒋图像数据画到界面上
private lateinit var screenFilter: ScreenFilter
private lateinit var surfaceTexture: SurfaceTexture
private var textureId: Int = 0
private var matrix: FloatArray = FloatArray(16)
constructor(glSurfaceView: GLSurfaceView) {
Log.d(TAG, "constructor: ")
this.glSurfaceView = glSurfaceView
context = glSurfaceView.context
cameraUtil = CameraUtil(context)
//设置版本
this.glSurfaceView.setEGLContextClientVersion(2)
this.glSurfaceView.setRenderer(this)
//当有数据来就更新界面,即调用glSurfaceView.requestRender()就会触发调用onDrawFrame来更新界面
this.glSurfaceView.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY
Log.d(TAG, "constructor: end")
}
override fun onDrawFrame(gl: GL10?) {
Log.d(TAG, "onDrawFrame: ")
//清除上一次数据
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
//更新surfaceTexture数据
surfaceTexture.updateTexImage()
surfaceTexture.getTransformMatrix(matrix)
fboFilter.setUniforms(matrix)
//将textureId对应纹理绘制到FBO中
// 这里一定要是局部变量或者另一个变量,因为如果在这里赋值改变了textureId,下一次执行onDrawFrame时,textureId的值就不对了
var textureId = fboFilter.onDrawFrame(textureId)
// 纹理绘制到画面上
screenFilter.onDrawFrame(textureId)
}
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
Log.d(TAG, "onSurfaceChanged: $width $height")
GLES20.glViewport(0, 0, width, height)
//设置surfaceTexture宽高
surfaceTexture.setDefaultBufferSize(width, height)
//创建FBO
fboFilter.fbo = FboFilter.FBO(width, height)
}
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
Log.d(TAG, "onSurfaceCreated: ")
GLES20.glClearColor(1f, 1f, 0f, 1f)
// 生成一个纹理
val textureIds = IntArray(1)
GLES20.glGenTextures(textureIds.size, textureIds, 0)
textureId = textureIds[0]
//使用textureId创建一个SurfaceTexture,预览的时候使用这个SurfaceTexture
surfaceTexture = SurfaceTexture(textureId)
//为surfaceTexture设置监听,当预览数据更新的时候,就会触发onFrameAvailable回调
surfaceTexture.setOnFrameAvailableListener(this)
fboFilter = FboFilter(context)
screenFilter = ScreenFilter(context)
}
/**
* 预览
*/
suspend fun startPreview(cameraId: String) {
//将cameraId对应摄像头的数据在surfaceTexture上显示
val outputs = listOf(Surface(surfaceTexture))
cameraUtil.startPreview(cameraId, outputs)
}
fun stopPreview() {
cameraUtil.release()
}
/**
* 摄像头新的一帧达到,更新glSurfaceView的界面
*/
override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) {
glSurfaceView.requestRender()
}
}
现在调用渲染器中的startPreview就可以进行预览了