Android自定义camera2相机系列（四）Opengles 预览和拍照进行实时处理
前面的博客，进行了部分的GLSL的语法学习,这一篇文章主要讲述了本人在开发 Camera2 + GLSurfaceView + GLSL 的开发过程的记录。如有错误还望指正。
此文章部分内容都基于 Android 自定义camera2 相机 （二）中的 camera2 相机打开 设置预览 绑定 等操作，如果有不懂 可以回到第二篇相关系列 文章中进行部分知识api 的学习。
Github 地址
Github 主要类地址
我们先看效果图，我这里只是在片元着色器中对R通道的 黑色进行了判断。当然如果有需求是 黑白图 则可以通过 1-R，1-G，1-B 来获得黑白图。切记GLSL中颜色的范围是 0-1 ，而Rgb中 范围则是 0-255.
1. 布局添加
在布局中 我们只添加了 一个Fragment ，我们需要通过动态添加的方式 将 GLSurfaceview 添加到主界面中。
2. 获取屏幕尺寸
通过 setUpCameraOutputs方法，设置 相机的配置属性，并返回相机的size，以便达到我们的全屏照相机需求。
/**
* 设置与摄像头相关的成员变量。
* @param width 摄像机预览的可用大小宽度
* @param height 相机预览的可用尺寸高度
*/
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public Size setUpCameraOutputs(int width, int height) {
mFile = new File(mActivity.getExternalFilesDir(null), "pic.png");
CameraManager manager = (CameraManager) mActivity.getSystemService(Context.CAMERA_SERVICE);
try {
for (String cameraId : manager.getCameraIdList()) {
CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
// 不使用前置摄像头
Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING);
if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) {
continue;
}
StreamConfigurationMap map = characteristics.get(
CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
if (map == null) {
continue;
}
// 静态图像捕获，选择最大可用大小。
Size largest = Collections.max(
Arrays.asList(map.getOutputSizes(ImageFormat.JPEG)),
new CompareSizesByArea());
mImageReader = ImageReader.newInstance(largest.getWidth(),
largest.getHeight(), ImageFormat.JPEG, 2);
mImageReader.setOnImageAvailableListener(
mOnImageAvailableListener, mBackgroundHandler);
//了解我们是否需要交换尺寸以获得相对于传感器的预览尺寸
int displayRotation = mActivity.getWindowManager().getDefaultDisplay().getRotation();
mSensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
boolean swappedDimensions = false;
switch (displayRotation) {
case Surface.ROTATION_0:
case Surface.ROTATION_180:
if (mSensorOrientation == 90 || mSensorOrientation == 270) {
swappedDimensions = true;
}
break;
case Surface.ROTATION_90:
case Surface.ROTATION_270:
if (mSensorOrientation == 0 || mSensorOrientation == 180) {
swappedDimensions = true;
}
break;
default:
Log.e(TAG, "Display rotation is invalid: " + displayRotation);
}
Point displaySize = new Point();
mActivity.getWindowManager().getDefaultDisplay().getSize(displaySize);
int rotatedPreviewWidth = width;
int rotatedPreviewHeight = height;
int maxPreviewWidth = displaySize.x;
int maxPreviewHeight = displaySize.y;
if (swappedDimensions) {
rotatedPreviewWidth = height;
rotatedPreviewHeight = width;
maxPreviewWidth = displaySize.y;
maxPreviewHeight = displaySize.x;
}
if (maxPreviewWidth > MAX_PREVIEW_WIDTH) {
maxPreviewWidth = MAX_PREVIEW_WIDTH;
}
if (maxPreviewHeight > MAX_PREVIEW_HEIGHT) {
maxPreviewHeight = MAX_PREVIEW_HEIGHT;
}
mPreviewSize = chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class),
rotatedPreviewWidth, rotatedPreviewHeight, maxPreviewWidth,
maxPreviewHeight, largest);
// 将TextureView的宽高比与我们选择的预览大小相匹配。
int orientation = mActivity.getResources().getConfiguration().orientation;
// 检查 远光灯
Boolean available = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE);
mFlashSupported = available == null ? false : available;
mCameraId = cameraId;
}
} catch (CameraAccessException e) {
e.printStackTrace();
} catch (NullPointerException ignored) {
}
return mPreviewSize;
}
通过 返回的 size，我们设置 CameraV2GLSurfaceView 布局的宽高。
public class CameraV2GLSurfaceView extends GLSurfaceView {
public static boolean shouldTakePic = false;
public void init(CameraV2 camera, boolean isPreviewStarted, Context context) {
setEGLContextClientVersion(2);
CameraV2Renderer mCameraV2Renderer = new CameraV2Renderer();
mCameraV2Renderer.init(this, camera, isPreviewStarted, context);
setRenderer(mCameraV2Renderer);
setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
}
public CameraV2GLSurfaceView(Context context) {
super(context);
}
}
继承GLSurfaceView 后，我们就需要 设置 Renderer 。当然这里的 cameraV2是拍照的集成工具类。我们在Renderer的onSurfaceCreated方法中创建一个OES纹理。
/**
* GLSurfaceView 创建
*
* @param gl GL10
* @param config EGLConfig
*/
@Override
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
// 创建纹理 返回 纹理ID
mOESTextureId = Utils.createOESTextureObject();
// 配置滤镜 加载 定点 和 片元 着色器
FilterEngine mFilterEngine = new FilterEngine(mOESTextureId, mContext);
mDataBuffer = mFilterEngine.getBuffer();
mShaderProgram = mFilterEngine.getShaderProgram();
glGenFramebuffers(1, mFBOIds, 0);
glBindFramebuffer(GL_FRAMEBUFFER, mFBOIds[0]);
// 获取顶点 和 片元 着色器中变量内容
uColorType = glGetUniformLocation(mShaderProgram, FilterEngine.COLOR_TYPE);
hChangeColor = GLES20.glGetUniformLocation(mShaderProgram, "vChangeColor");
hChangeColor2 = GLES20.glGetUniformLocation(mShaderProgram, "vChangeColorB");
hChangeColor3 = GLES20.glGetUniformLocation(mShaderProgram, "vChangeColorC");
hArraySize = GLES20.glGetUniformLocation(mShaderProgram, "vArraysSize");
}
之后根据OES纹理Id创建SurfaceTexture，用来接收Camera2的预览数据
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
private boolean initSurfaceTexture() {
if (mCamera == null || mCameraV2GLSurfaceView == null) {
Log.i(TAG, "mCamera or mGLSurfaceView is null!");
return false;
}
// 根据 oesId 创建 SurfaceTexture
mSurfaceTexture = new SurfaceTexture(mOESTextureId);
mSurfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
@Override
public void onFrameAvailable(SurfaceTexture surfaceTexture) {
// 每获取到一帧数据时请求OpenGL ES进行渲染
mCameraV2GLSurfaceView.requestRender();
}
});
//讲此SurfaceTexture作为相机预览输出 （相互绑定）
mCamera.setPreviewTexture(mSurfaceTexture);
mCamera.createCameraPreviewSession();
return true;
}
最后初始化OpenGL ES环境，包括 shader编写 和 编译，链接到program。（由于这里内容比较多，在文章末尾我会附上本人 github demo 地址）
/**
* 滤镜 工具
* 参考url ： [https://blog.csdn.net/lb377463323/article/details/78054892]
* @date 2019年2月12日 14:10:07
* @author ymc
*/
public class FilterEngine {
@SuppressLint("StaticFieldLeak")
private static FilterEngine filterEngine = null;
private Context mContext;
/**
* 存放顶点的Color数组
*/
private FloatBuffer mBuffer;
private int mOESTextureId = -1;
private int vertexShader = -1;
private int fragmentShader = -1;
private int mShaderProgram = -1;
private int aPositionLocation = -1;
private int aTextureCoordLocation = -1;
private int uTextureMatrixLocation = -1;
private int uTextureSamplerLocation = -1;
/**
* 每行前两个值为顶点坐标，后两个为纹理坐标
*/
private static final float[] VERTEX_DATA = {
1f, 1f, 1f, 1f,
-1f, 1f, 0f, 1f,
-1f, -1f, 0f, 0f,
1f, 1f, 1f, 1f,
-1f, -1f, 0f, 0f,
1f, -1f, 1f, 0f
};
public static final String POSITION_ATTRIBUTE = "aPosition";
public static final String TEXTURE_COORD_ATTRIBUTE = "aTextureCoordinate";
public static final String TEXTURE_MATRIX_UNIFORM = "uTextureMatrix";
public static final String TEXTURE_SAMPLER_UNIFORM = "uTextureSampler";
public static final String COLOR_TYPE = "vColorType";
/**
* 构造方法
* @param oestextureid oes id
* @param context 上下文
*/
public FilterEngine(int oestextureid, Context context) {
mContext = context;
mOESTextureId = oestextureid;
mBuffer = createBuffer(VERTEX_DATA);
/**
* 预览相机的着色器，顶点着色器不变，需要修改片元着色器，不再用sampler2D采样，
* 需要使用samplerExternalOES 纹理采样器，并且要在头部增加使用扩展纹理的声明
* #extension GL_OES_EGL_image_external : require。
*/
fragmentShader = loadShader(GL_FRAGMENT_SHADER, Utils.readShaderFromResource(mContext, R.raw.base_fragment_shader));
vertexShader = loadShader(GL_VERTEX_SHADER, Utils.readShaderFromResource(mContext, R.raw.base_vertex_shader));
mShaderProgram = linkProgram(vertexShader, fragmentShader);
}
/**
* 创建 FloatBuffer 数组 (防止内存回收)
* @param vertexData float 数组
* @return FloatBuffer
*/
private FloatBuffer createBuffer(float[] vertexData) {
FloatBuffer buffer = ByteBuffer.allocateDirect(vertexData.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer();
buffer.put(vertexData, 0, vertexData.length).position(0);
return buffer;
}
/**
* 加载着色器
* GL_VERTEX_SHADER 代表生成顶点着色器
* GL_FRAGMENT_SHADER 代表生成片段着色器
* @param type 类型
* @param shaderSource shader string
* @return shader
*/
private int loadShader(int type, String shaderSource) {
int shader = glCreateShader(type);
if (shader == 0) {
throw new RuntimeException("Create Shader Failed!" + glGetError());
}
glShaderSource(shader, shaderSource);
glCompileShader(shader);
return shader;
}
/**
* 将两个Shader链接至program中
* @param verShader verShader
* @param fragShader fragShader
* @return program
*/
private int linkProgram(int verShader, int fragShader) {
int program = glCreateProgram();
if (program == 0) {
throw new RuntimeException("Create Program Failed!" + glGetError());
}
//附着顶点和片段着色器
glAttachShader(program, verShader);
glAttachShader(program, fragShader);
// 绑定 program
glLinkProgram(program);
//告诉OpenGL ES使用此program
glUseProgram(program);
return program;
}
public void drawTexture(float[] transformMatrix) {
aPositionLocation = glGetAttribLocation(mShaderProgram, FilterEngine.POSITION_ATTRIBUTE);
aTextureCoordLocation = glGetAttribLocation(mShaderProgram, FilterEngine.TEXTURE_COORD_ATTRIBUTE);
uTextureMatrixLocation = glGetUniformLocation(mShaderProgram, FilterEngine.TEXTURE_MATRIX_UNIFORM);
uTextureSamplerLocation = glGetUniformLocation(mShaderProgram, FilterEngine.TEXTURE_SAMPLER_UNIFORM);
glActiveTexture(GLES20.GL_TEXTURE0);
glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mOESTextureId);
glUniform1i(uTextureSamplerLocation, 0);
glUniformMatrix4fv(uTextureMatrixLocation, 1, false, transformMatrix, 0);
if (mBuffer != null) {
mBuffer.position(0);
glEnableVertexAttribArray(aPositionLocation);
glVertexAttribPointer(aPositionLocation, 2, GL_FLOAT, false, 16, mBuffer);
mBuffer.position(2);
glEnableVertexAttribArray(aTextureCoordLocation);
glVertexAttribPointer(aTextureCoordLocation, 2, GL_FLOAT, false, 16, mBuffer);
glDrawArrays(GL_TRIANGLES, 0, 6);
}
}
...
set
....
get
...
}
上述工具类中 加载了shader 并且将 shader 绑定到 program中，接下来我们设置 顶单着色器
attribute vec4 aPosition;
uniform mat4 uTextureMatrix;
attribute vec4 aTextureCoordinate;
varying vec2 vTextureCoord;
void main()
{
vTextureCoord = (uTextureMatrix * aTextureCoordinate).xy;
gl_Position = aPosition;
}
设置 片元着色器 glsl 代码，下边我会写多种滤镜效果
#extension GL_OES_EGL_image_external : require
uniform samplerExternalOES uTextureSampler;
precision mediump float;
uniform int vColorType;
varying vec2 vTextureCoord;
uniform int vArraysSize;
uniform vec3 vChangeColor;
uniform vec3 vChangeColorB;
uniform vec3 vChangeColorC;
float debugFloatA;
float debugFloatB;
void main()
{
// 本人 demo 调试效果 将R 通道的 黑色设置为 白色
vec4 vCameraColor = texture2D(uTextureSampler, vTextureCoord);
gl_FragColor = vec4(vCameraColor.r, vCameraColor.g, vCameraColor.b, 1.0);
for(int i = 0;i打开相机
/**
* 打开相机
*
* @return boolean
*/
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public boolean openCamera() {
CameraManager cameraManager = (CameraManager) mActivity.getSystemService(Context.CAMERA_SERVICE);
try {
if (ActivityCompat.checkSelfPermission(mActivity, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
return false;
}
cameraManager.openCamera(mCameraId, mStateCallback, mBackgroundHandler);
if (!mCameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
throw new RuntimeException("Time out waiting to lock camera opening.");
}
} catch (CameraAccessException | InterruptedException e) {
e.printStackTrace();
return false;
}
return true;
}
相机状态改变回调
/**
* CameraDevice 改变状态时候 调用
*/
private CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
@Override
public void onOpened(@NonNull CameraDevice camera) {
mCameraDevice = camera;
//打开相机时会调用此方法。 我们在这里开始相机预览。
mCameraOpenCloseLock.release();
// createCameraPreviewSession();
}
@Override
public void onDisconnected(@NonNull CameraDevice camera) {
mCameraOpenCloseLock.release();
camera.close();
mCameraDevice = null;
}
@Override
public void onError(@NonNull CameraDevice camera, int error) {
mCameraOpenCloseLock.release();
camera.close();
mCameraDevice = null;
mActivity.finish();
}
};
拍照 静态锁定
/**
* 拍摄静止图片。 当我们得到响应时，应该调用此方法
*/
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
private void captureStillPicture() {
try {
if (null == mActivity || null == mCameraDevice) {
return;
}
final CaptureRequest.Builder captureBuilder =
mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
captureBuilder.addTarget(mImageReader.getSurface());
// 使用与预览相同的AE和AF模式。
captureBuilder.set(CaptureRequest.CONTROL_AF_MODE,
CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
// 查看使用支持 flash
if (mFlashSupported) {
captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
}
int rotation = mActivity.getWindowManager().getDefaultDisplay().getRotation();
captureBuilder.set(CaptureRequest.JPEG_ORIENTATION, getOrientation(rotation));
CameraCaptureSession.CaptureCallback capturecallback
= new CameraCaptureSession.CaptureCallback() {
@Override
public void onCaptureCompleted(@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request,
@NonNull TotalCaptureResult result) {
Log.e(TAG, "--------------------:" + mFile.toString());
unlockFocus();
}
};
mCaptureSession.stopRepeating();
mCaptureSession.abortCaptures();
mCaptureSession.capture(captureBuilder.build(), capturecallback, mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
Log.e(TAG, "capture err: " + e.getMessage());
}
}
/**
* 解锁焦点 在静止图像捕获序列时调用此方法
*/
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
private void unlockFocus() {
try {
mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AF_TRIGGER,
CameraMetadata.CONTROL_AF_TRIGGER_CANCEL);
if (mFlashSupported) {
mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AE_MODE,
CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
}
mCaptureSession.capture(mPreviewRequestBuilder.build(), mCaptureCallback,
mBackgroundHandler);
// 相机转为正常状态
mState = STATE_PREVIEW;
mCaptureSession.setRepeatingRequest(mPreviewRequest, mCaptureCallback,
mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
然后OpenGL ES将此OES纹理数据绘制到屏幕上，我们就可以预览到 通过 片元着色器修改过后的预览效果，但是这里注意如果 这里只是预览的效果改变，但是 拍照后得到的图片还是 原图，本人这里暂时用的方法是 截图的方式，就是获取预览
流中的 数据后 循环所有像素，重绘保存为图片即可。
在 Rander 中 的 onDrawFrame() 方法中 ，
/**
* 根据标识 是否截图
* 参考url： [http://hounychang.github.io/2015/05/13/%E5%AF%B9GLSurfaceView%E6%88%AA%E5%9B%BE/]
*/
if (CameraV2GLSurfaceView.shouldTakePic) {
CameraV2GLSurfaceView.shouldTakePic = false;
// bindfbo();
int w = surfaceWidth;
int h = surfaceHeight;
int b[] = new int[w * h];
int bt[] = new int[w * h];
IntBuffer buffer = IntBuffer.wrap(b);
buffer.position(0);
GLES20.glReadPixels(0, 0, w, h, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
int pix = b[i * w + j];
int pb = (pix >> 16) & 0xff;
int pr = (pix << 16) & 0x00ff0000;
int pix1 = (pix & 0xff00ff00) | pr | pb;
bt[(h - i - 1) * w + j] = pix1;
}
}
Bitmap inBitmap;
inBitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888);
//为了图像能小一点，使用了RGB_565而不是ARGB_8888
inBitmap.copyPixelsFromBuffer(buffer);
inBitmap = Bitmap.createBitmap(bt, w, h, Bitmap.Config.ARGB_8888);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
inBitmap.compress(Bitmap.CompressFormat.PNG, 90, bos);
byte[] bitmapData = bos.toByteArray();
ByteArrayInputStream fis = new ByteArrayInputStream(bitmapData);
File mFile = new File(mContext.getExternalFilesDir(null), "pic1.png");
try {
FileOutputStream fos = new FileOutputStream(mFile);
byte[] buf = new byte[1024];
int len;
while ((len = fis.read(buf)) > 0) {
fos.write(buf, 0, len);
}
fis.close();
fos.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
//旋转角度
// int rotate = BitmapRotating.readPictureDegree(mFile.getPath());
// BitmapRotating.rotaingImageView(rotate,inBitmap);
inBitmap.recycle();
// unbindfbo();
}
}
long t2 = System.currentTimeMillis();
long t = t2 - t1;
Log.i(TAG, "onDrawFrame: time: " + t);
到这里上个月的 简单的 项目demo 讲解已经基本完毕。如有错误 还望读者指出，本人小白，不断学习中…