相机之使用OpenGL预览
相机之使用OpenGL拍照
相机之使用OpenGL录像
添加音频
步骤
- 创建音频格式 MediaFormat
- 创建 MediaCodec 音频编码器
- 初始化 AudioRecord ，并调用 startRecording() 开始录制音频
- 在线程中使用read(buffer, BUFFER_SIZE)方法读取音频
- 将读取到的音频数据放入 MediaCodec 的输入缓冲区中
- 在 MediaCodec 输出缓冲区使用 MediaMuxer 和视频一起封装到MP4
注意点
- 一定要设置时间戳bufferInfo.presentationTimeUs，否则音视频不同步
- MediaMuxer 的 start() 和 release() 只能调用一次，而编码的时候又必须将音频和视频两个格式轨道addTrack进MediaMuxer 之后，才能调用start()，结束的时候也需要音视频都结束编码才能release() 。我这里采用了CyclicBarrier来处理，他可以计数到达TRACK_COUNT后，线程才能继续运行，其第二个参数作用是：最后一个到达线程要做的任务
- 视频可以通过mediaCodec.signalEndOfInputStream()结束录制，但音频需要mediaCodec.queueInputBuffer传入结束标志BUFFER_FLAG_END_OF_STREAM
视频和音频共用部分
open class BaseRecorder(val mediaMuxer: MediaMuxer) {
var isStart = false
lateinit var mediaCodec: MediaCodec
var trackIndex: Int = 0
private var prePtsUs: Long = 0
companion object {
private const val TAG = "BaseRecorder"
}
/**
* 计算数据显示的时间戳
*/
fun getPtsUs(): Long {
var result = System.nanoTime() / 1000L
if (result < prePtsUs) {
result += (prePtsUs - result)
}
prePtsUs = result
return result
}
/**
* 将编码后的数据写入Muxer，生成MP4文件
*/
open fun writeToMuxer(endOfStream: Boolean) {
var bufferInfo = MediaCodec.BufferInfo()
loop@ while (true) {
//得到当前编码器的状态
var status = mediaCodec.dequeueOutputBuffer(bufferInfo, 10_000)
// Log.d(TAG, "writeToMuxer: status=$status")
when (status) {
//稍后再试，直接退出循环，直到下次调用writeToMuxer
MediaCodec.INFO_TRY_AGAIN_LATER -> {
if (!endOfStream) {
break@loop
}
}
//格式变化，为mediaMuxer添加轨道，一共两个轨道，一个音频，一个视频，如果都添加了，就可以开始封装为MP4了
MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
var outputFormat = mediaCodec.outputFormat
//添加格式轨道
trackIndex = mediaMuxer.addTrack(outputFormat)
Log.d(TAG, "writeToMuxer: currentThread=${Thread.currentThread().name}")
MediaRecorder.startMuxer.await()
}
MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED -> {
Log.d(TAG, "getCodec: INFO_OUTPUT_BUFFERS_CHANGED")
}
else -> {
//得到编码好的数据
var outputBuffer = mediaCodec.getOutputBuffer(status)
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
bufferInfo.size = 0
}
//数据大小不等于0
if (bufferInfo.size != 0) {
//设置数据显示的时间戳
bufferInfo.presentationTimeUs = getPtsUs()
outputBuffer.position(bufferInfo.offset)
outputBuffer.limit(bufferInfo.offset + bufferInfo.size)
//将编码后的数据写入相应轨道
mediaMuxer.writeSampleData(trackIndex, outputBuffer, bufferInfo)
}
//释放Buffer,以便复用
mediaCodec.releaseOutputBuffer(status, false)
//此次编码完成，退出
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
Log.d(TAG, "writeToMuxer: BUFFER_FLAG_END_OF_STREAM break@loop")
MediaRecorder.stopMuxer.await()
break@loop
}
}
}
}
}
}
AudioRecorder
class AudioRecorder(
mediaMuxer: MediaMuxer
) : BaseRecorder(mediaMuxer) {
private var recordHandler: Handler
private lateinit var audioRecord: AudioRecord
companion object {
private const val TAG = "AudioRecorder"
private const val SAMPLE_RATE_IN_HZ = 44100
const val BIT_RATE = 64000
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
private const val TIMEOUT_US = 1000000000L
val BUFFER_SIZE: Int by lazy {
AudioRecord.getMinBufferSize(
SAMPLE_RATE_IN_HZ,
CHANNEL_CONFIG,
AUDIO_FORMAT
)
}
}
init {
//创建音频格式，参数对应：mime type、采样率、声道数
var audioFormat = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC,
SAMPLE_RATE_IN_HZ, 1
)
audioFormat.setInteger(
MediaFormat.KEY_AAC_PROFILE,
MediaCodecInfo.CodecProfileLevel.AACObjectLC
)
//设置比特率
audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1)
audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, CHANNEL_CONFIG)
//创建音频编码器mediaCodec
mediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
//配置音频编码器mediaCodec
mediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
//在线程中录制音频
var recordThread = HandlerThread("audioThread").apply { start() }
recordHandler = Handler(recordThread.looper)
}
/**
* 开始录制音频
*/
fun start() {
//使用AudioRecord进行录制
audioRecord = AudioRecord(
MediaRecorder.AudioSource.MIC,
SAMPLE_RATE_IN_HZ,
CHANNEL_CONFIG,
AUDIO_FORMAT,
BUFFER_SIZE
)
if (audioRecord.state != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "start: audioRecord init failed!")
return
}
audioRecord.startRecording()
isStart = true
recordHandler.post {
mediaCodec.start()
var buffer = ByteBuffer.allocateDirect(BUFFER_SIZE)
while (isStart) {
//将音频数据读取到buffer中
var readLength = audioRecord.read(buffer, BUFFER_SIZE)
buffer.position(readLength)
buffer.flip()
//将读取到的音频数据写到mediaCodec中
inputDataToCodec(buffer, readLength)
writeToMuxer(false)
}
}
}
/**
* 将AudioRecord读到的音频数据放入mediaCodec
*/
private fun inputDataToCodec(buffer: ByteBuffer?, readLength: Int) {
//获取一个可用的InputBuffer的索引
var index = mediaCodec.dequeueInputBuffer(TIMEOUT_US)
Log.d(TAG, "prepareDataForCodec: $index")
//如果读取的音频数据长等于0，说明没有数据，结束编码
if (readLength <= 0) {
mediaCodec.queueInputBuffer(
index,
0,
0,
System.nanoTime() / 1000L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
} else if (index >= 0) {
//将音频数据写入mediaCodec，进行编码
var inputBuffer = mediaCodec.getInputBuffer(index)
inputBuffer.put(buffer)
mediaCodec.queueInputBuffer(
index,
0,
readLength,
System.nanoTime() / 1000L,
0
)
}
}
/**
* 将音频流封装进Muxer
*/
override fun writeToMuxer(endOfStream: Boolean) {
Log.e(TAG, "writeToMuxer: endOfStream=$endOfStream")
if (endOfStream) {
//结束音频录制，因此长度写入0
inputDataToCodec(null, 0)
// mediaCodec.signalEndOfInputStream()
}
//调用父类的方法，将mediaCodec编码后的数据写入Muxer
super.writeToMuxer(endOfStream)
}
/**
* 结束录音
*/
fun stop() {
isStart = false
recordHandler.post {
//写入标志位，停止录制
writeToMuxer(true)
//释放各种资源
mediaCodec.stop()
mediaCodec.release()
audioRecord.stop()
audioRecord.release()
recordHandler.looper.quitSafely()
}
}
}
VideoRecorder
class VideoRecorder(
private val context: Context,
mediaMuxer: MediaMuxer,
private var width: Int,
private val height: Int,
private val eglContext: EGLContext
) : BaseRecorder(mediaMuxer) {
companion object {
private const val TAG = "VideoRecorder"
const val FRAME_RATE = 25
const val I_FRAME_INTERVAL = 10
}
private lateinit var eglBase: EglBase
private lateinit var recordHandler: Handler
fun start() {
//创建视频格式
var videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height)
videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, calculateBitRate())
//CQ 完全不控制码率，尽最大可能保证图像质量
//CBR 编码器会尽量把输出码率控制为设定值
//VBR 编码器会根据图像内容的复杂度（实际上是帧间变化量的大小）来动态调整输出码率，图像复杂则码率高，图像简单则码率低
videoFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR)
//设置帧率
videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE)
//设置I帧的间隔
videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL)
//颜色格式是GraphicBuffer元数据
videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
//创建视频编码器mediaCodec
mediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
//配置视频编码器mediaCodec的格式
mediaCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
//通过编码器创建一个Surface，之后的图像数据绘制到这上面，再进行保存
var inputSurface = mediaCodec.createInputSurface()
//为了不阻塞，录制在单开的一个线程中进行
var recordThread = HandlerThread("videoThread").apply { start() }
recordHandler = Handler(recordThread.looper)
recordHandler.post {
eglBase = EglBase(context, eglContext, inputSurface, width, height)
mediaCodec.start()
}
isStart = true
}
/**
* 计算码率
*/
private fun calculateBitRate(): Int {
val bitrate = (0.25f * FRAME_RATE * width * height).toInt()
Log.d(TAG, "calculateBitRate: bitrate=$bitrate")
return bitrate
}
/**
* 编码这一帧数据
*/
fun encodeFrame(textureId: Int, timestamp: Long) {
//没有开始录制，直接返回
if (!isStart) {
return
}
recordHandler.post {
if (isStart) {
Log.e(TAG, "encodeFrame: timestamp=$timestamp isStart=$isStart")
//在eglBase中绘制出这一帧内容
eglBase.draw(textureId, timestamp)
//将这一帧封装进Muxer
writeToMuxer(false)
}
}
}
/**
* 将视频流封装进Muxer
*/
override fun writeToMuxer(endOfStream: Boolean) {
Log.e(TAG, "writeToMuxer: endOfStream=$endOfStream")
if (endOfStream) {
mediaCodec.signalEndOfInputStream()
}
//调用父类的方法，将mediaCodec编码后的数据写入Muxer
super.writeToMuxer(endOfStream)
}
/**
* 停止录制
*/
fun stop() {
isStart = false
recordHandler.post {
//写入标志位，停止录制
writeToMuxer(true)
//释放各种资源
eglBase.release()
recordHandler.looper.quitSafely()
}
}
}
渲染器
class GlRenderer : GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
companion object {
private const val TAG = "MyRenderer"
}
private var width: Int=0
private var height: Int=0
//EGL上下文，录像的时候使用
private lateinit var eglContext: EGLContext
private val glSurfaceView: GLSurfaceView
private val context: Context
//用于控制摄像头，打开摄像头之类的
private var cameraUtil: CameraUtil
//将摄像头数据画到FBO中
private lateinit var fboFilter: FboFilter
//蒋图像数据画到界面上
private lateinit var screenFilter: ScreenFilter
private lateinit var surfaceTexture: SurfaceTexture
private var textureId: Int = 0
private var matrix: FloatArray = FloatArray(16)
//录像的工具
private var mediaRecorder: MediaRecorder? = null
constructor(glSurfaceView: GLSurfaceView) {
Log.d(TAG, "constructor: ")
this.glSurfaceView = glSurfaceView
context = glSurfaceView.context
cameraUtil = CameraUtil(context)
//设置版本
this.glSurfaceView.setEGLContextClientVersion(2)
this.glSurfaceView.setRenderer(this)
//当有数据来就更新界面，即调用glSurfaceView.requestRender()就会触发调用onDrawFrame来更新界面
this.glSurfaceView.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY
Log.d(TAG, "constructor: end")
}
override fun onDrawFrame(gl: GL10?) {
Log.d(TAG, "onDrawFrame: ")
//清除上一次数据
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
//更新surfaceTexture数据
surfaceTexture.updateTexImage()
surfaceTexture.getTransformMatrix(matrix)
fboFilter.setUniforms(matrix)
//将textureId对应纹理绘制到FBO中
// 这里一定要是局部变量或者另一个变量，因为如果在这里赋值改变了textureId，下一次执行onDrawFrame时，textureId的值就不对了
var textureId = fboFilter.onDrawFrame(textureId)
//fobTextureId纹理绘制到画面上
screenFilter.onDrawFrame(textureId)
//如果当前正在录制的话，将fobTextureId纹理编码
mediaRecorder?.encodeFrame(textureId, surfaceTexture.timestamp)
}
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
Log.d(TAG, "onSurfaceChanged: $width $height")
GLES20.glViewport(0, 0, width, height)
//设置surfaceTexture宽高
surfaceTexture.setDefaultBufferSize(width, height)
//摄像头不支持奇数的宽高
this.width = if ((width and 1) == 1) width - 1 else width
this.height = if ((height and 1) == 1) height - 1 else height
fboFilter = FboFilter(context, width, height)
screenFilter = ScreenFilter(context, width, height)
eglContext = EGL14.eglGetCurrentContext()
}
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
Log.d(TAG, "onSurfaceCreated: ")
GLES20.glClearColor(1f, 1f, 0f, 1f)
// 生成一个纹理
val textureIds = IntArray(1)
GLES20.glGenTextures(textureIds.size, textureIds, 0)
textureId = textureIds[0]
//使用textureId创建一个SurfaceTexture，预览的时候使用这个SurfaceTexture
surfaceTexture = SurfaceTexture(textureId)
//为surfaceTexture设置监听，当预览数据更新的时候，就会触发onFrameAvailable回调
surfaceTexture.setOnFrameAvailableListener(this)
}
/**
* 预览
*/
suspend fun startPreview(cameraId: String) {
//将cameraId对应摄像头的数据在surfaceTexture上显示
val outputs = listOf(Surface(surfaceTexture))
cameraUtil.startPreview(cameraId, outputs)
}
fun stopPreview() {
cameraUtil.release()
}
/**
* 摄像头新的一帧达到，更新glSurfaceView的界面
*/
override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) {
glSurfaceView.requestRender()
}
/**
* 设置照片回调，在screenFilter画到屏幕后，进行回调保存当前帧
*/
fun takePicture(pictureCallBack: (Bitmap) -> Unit) {
screenFilter.setSaveFrame(pictureCallBack)
}
/**
* 开始录像
*/
fun startRecord(path: String) {
mediaRecorder = MediaRecorder(
glSurfaceView.context,
path,
width,
height,
eglContext
)
mediaRecorder?.start()
}
/**
* 停止录像
*/
fun stopRecord() {
mediaRecorder?.stop()
}
}