google MediaPipe Hands手势识别横屏显示-解决CameraXPreviewHelper不起作用

 

 

我用的项目版本是已经编译好的版本-连接:   https://github.com/mgyong/awesome-mediapipe

kotlin版本-连接:https://github.com/machidyo/MediaPipeHandTracking

业务需要横屏,大神回答将纹理进行旋转

https://github.com/google/mediapipe/issues/568

wm : WindowManager
converter: ExternalTextureConverter

rotation = WindowManager.getDefaultDisplay().getRotation()
converter.setRotation((360 - rotation) % 360)

Then in ExternalTextureRenderer.java

//ADD
  private int rotation = 0;
  public void setRotation(int rotation) {
    this.rotation = rotation;
  }
//ADD

  public void render(SurfaceTexture surfaceTexture) {
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    ShaderUtil.checkGlError("glActiveTexture");
    surfaceTexture.updateTexImage(); // This implicitly binds the texture.
    surfaceTexture.getTransformMatrix(textureTransformMatrix);
//ADD
    switch (rotation) {
      case 0:
        break;
      case 90:
        Matrix.rotateM(textureTransformMatrix, 0, 90, 0, 0, 1);
        Matrix.translateM(textureTransformMatrix, 0, 0, -1, 0);
        break;
      case 180:
        Matrix.rotateM(textureTransformMatrix, 0, 180, 0, 0, 1);
        Matrix.translateM(textureTransformMatrix, 0, -1, -1, 0);
        break;
      case 270:
        Matrix.rotateM(textureTransformMatrix, 0, 270, 0, 0, 1);
        Matrix.translateM(textureTransformMatrix, 0, -1, 0, 0);
        break;
      default:
        //unknown
    }
//ADD

 

不过以上代码都在aar里面  我试了一下 具体代码如下

在AndroidManifest.xml里添加横屏属性 MainActivity  android:screenOrientation="landscape"

新建CustomExternalTextureRenderer.java代码如下

package com.example.mediapipehandtracking;

import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.opengl.Matrix;

import com.google.mediapipe.glutil.CommonShaders;
import com.google.mediapipe.glutil.ExternalTextureRenderer;
import com.google.mediapipe.glutil.ShaderUtil;

import java.nio.FloatBuffer;
import java.util.HashMap;
import java.util.Map;

public class CustomExternalTextureRenderer extends ExternalTextureRenderer {

    private static final FloatBuffer TEXTURE_VERTICES = ShaderUtil.floatBuffer(new float[]{0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F});
    private static final FloatBuffer FLIPPED_TEXTURE_VERTICES = ShaderUtil.floatBuffer(new float[]{0.0F, 1.0F, 1.0F, 1.0F, 0.0F, 0.0F, 1.0F, 0.0F});
    private static final String TAG = "ExternalTextureRend";
    private static final int ATTRIB_POSITION = 1;
    private static final int ATTRIB_TEXTURE_COORDINATE = 2;
    private int program = 0;
    private int frameUniform;
    private int textureTransformUniform;
    private float[] textureTransformMatrix = new float[16];
    private boolean flipY;

    public CustomExternalTextureRenderer() {
    }

    //ADD
    private int rotation = 0;
    public void setRotation(int rotation) {
        this.rotation = rotation;
    }

    public void render(SurfaceTexture surfaceTexture) {
        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        ShaderUtil.checkGlError("glActiveTexture");
        surfaceTexture.updateTexImage(); // This implicitly binds the texture.
        surfaceTexture.getTransformMatrix(textureTransformMatrix);


        switch (rotation) {
            case 0:
                break;
            case 90:
                Matrix.rotateM(textureTransformMatrix, 0, 90, 0, 0, 1);
                Matrix.translateM(textureTransformMatrix, 0, 0, -1, 0);
                break;
            case 180:
                Matrix.rotateM(textureTransformMatrix, 0, 180, 0, 0, 1);
                Matrix.translateM(textureTransformMatrix, 0, -1, -1, 0);
                break;
            case 270:
                Matrix.rotateM(textureTransformMatrix, 0, 270, 0, 0, 1);
                Matrix.translateM(textureTransformMatrix, 0, -1, 0, 0);
                break;
            default:
                //unknown
        }
        GLES20.glTexParameteri(36197, 10241, 9729);
        GLES20.glTexParameteri(36197, 10240, 9729);
        GLES20.glTexParameteri(36197, 10242, 33071);
        GLES20.glTexParameteri(36197, 10243, 33071);
        ShaderUtil.checkGlError("glTexParameteri");
        GLES20.glUseProgram(this.program);
        ShaderUtil.checkGlError("glUseProgram");
        GLES20.glUniform1i(this.frameUniform, 0);
        ShaderUtil.checkGlError("glUniform1i");
        GLES20.glUniformMatrix4fv(this.textureTransformUniform, 1, false, this.textureTransformMatrix, 0);
        ShaderUtil.checkGlError("glUniformMatrix4fv");
        GLES20.glEnableVertexAttribArray(1);
        GLES20.glVertexAttribPointer(1, 2, 5126, false, 0, CommonShaders.SQUARE_VERTICES);
        GLES20.glEnableVertexAttribArray(2);
        GLES20.glVertexAttribPointer(2, 2, 5126, false, 0, this.flipY ? FLIPPED_TEXTURE_VERTICES : TEXTURE_VERTICES);
        ShaderUtil.checkGlError("program setup");
        GLES20.glDrawArrays(5, 0, 4);
        ShaderUtil.checkGlError("glDrawArrays");
        GLES20.glBindTexture(36197, 0);
        ShaderUtil.checkGlError("glBindTexture");


        GLES20.glFinish();
    }
    public void setup() {
        Map attributeLocations = new HashMap();
        attributeLocations.put("position", 1);
        attributeLocations.put("texture_coordinate", 2);
        this.program = ShaderUtil.createProgram("uniform mat4 texture_transform;\nattribute vec4 position;\nattribute mediump vec4 texture_coordinate;\nvarying mediump vec2 sample_coordinate;\n\nvoid main() {\n  gl_Position = position;\n  sample_coordinate = (texture_transform * texture_coordinate).xy;\n}", "#extension GL_OES_EGL_image_external : require\nvarying mediump vec2 sample_coordinate;\nuniform samplerExternalOES video_frame;\n\nvoid main() {\n  gl_FragColor = texture2D(video_frame, sample_coordinate);\n}", attributeLocations);
        this.frameUniform = GLES20.glGetUniformLocation(this.program, "video_frame");
        this.textureTransformUniform = GLES20.glGetUniformLocation(this.program, "texture_transform");
        ShaderUtil.checkGlError("glGetUniformLocation");
    }

    public void setFlipY(boolean flip) {
        this.flipY = flip;
    }



    public void release() {
        GLES20.glDeleteProgram(this.program);
    }
}

创建CustomExternalTextureConverter.java 代码如下

package com.example.mediapipehandtracking;

import android.annotation.SuppressLint;
import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.util.Log;

import com.google.mediapipe.components.ExternalTextureConverter;
import com.google.mediapipe.components.TextureFrameConsumer;
import com.google.mediapipe.components.TextureFrameProducer;
import com.google.mediapipe.framework.AppTextureFrame;
import com.google.mediapipe.glutil.ExternalTextureRenderer;
import com.google.mediapipe.glutil.GlThread;
import com.google.mediapipe.glutil.ShaderUtil;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import javax.microedition.khronos.egl.EGLContext;

public class CustomExternalTextureConverter implements TextureFrameProducer {
    private static final String TAG = "ExternalTextureConv";
    private static final int DEFAULT_NUM_BUFFERS = 2;
    private static final String THREAD_NAME = "ExternalTextureConverter";
    private CustomExternalTextureConverter.RenderThread thread;

    public CustomExternalTextureConverter(EGLContext parentContext, int numBuffers,int rotation) {
        this.thread = new CustomExternalTextureConverter.RenderThread(parentContext, numBuffers,rotation);
        this.thread.setName("ExternalTextureConverter");
        this.thread.start();

        try {
            this.thread.waitUntilReady();
        } catch (InterruptedException var4) {
            Thread.currentThread().interrupt();
            Log.e("ExternalTextureConv", "thread was unexpectedly interrupted: " + var4.getMessage());
            throw new RuntimeException(var4);
        }
    }

    public void setFlipY(boolean flip) {
        this.thread.setFlipY(flip);
    }

    public CustomExternalTextureConverter(EGLContext parentContext) {
        this(parentContext, 2,0);
    }

    public CustomExternalTextureConverter(EGLContext parentContext, SurfaceTexture texture, int targetWidth, int targetHeight) {
        this(parentContext);
        this.thread.setSurfaceTexture(texture, targetWidth, targetHeight);
    }

    public void setSurfaceTexture(SurfaceTexture texture, int width, int height) {
        if (texture == null || width != 0 && height != 0) {
            this.thread.getHandler().post(() -> {
                this.thread.setSurfaceTexture(texture, width, height);
            });
        } else {
            throw new RuntimeException("ExternalTextureConverter: setSurfaceTexture dimensions cannot be zero");
        }
    }

    public void setSurfaceTextureAndAttachToGLContext(SurfaceTexture texture, int width, int height) {
        if (texture == null || width != 0 && height != 0) {
            this.thread.getHandler().post(() -> {
                this.thread.setSurfaceTextureAndAttachToGLContext(texture, width, height);
            });
        } else {
            throw new RuntimeException("ExternalTextureConverter: setSurfaceTexture dimensions cannot be zero");
        }
    }

    public void setConsumer(TextureFrameConsumer next) {
        this.thread.setConsumer(next);
    }

    public void addConsumer(TextureFrameConsumer consumer) {
        this.thread.addConsumer(consumer);
    }

    public void removeConsumer(TextureFrameConsumer consumer) {
        this.thread.removeConsumer(consumer);
    }

    public void close() {
        if (this.thread != null) {
            this.thread.getHandler().post(() -> {
                this.thread.setSurfaceTexture((SurfaceTexture)null, 0, 0);
            });
            this.thread.quitSafely();

            try {
                this.thread.join();
            } catch (InterruptedException var2) {
                Thread.currentThread().interrupt();
                Log.e("ExternalTextureConv", "thread was unexpectedly interrupted: " + var2.getMessage());
                throw new RuntimeException(var2);
            }
        }
    }

    private static class RenderThread extends GlThread implements SurfaceTexture.OnFrameAvailableListener {
        private static final long NANOS_PER_MICRO = 1000L;
        private volatile SurfaceTexture surfaceTexture = null;
        private final List consumers;
        private List outputFrames = null;
        private int outputFrameIndex = -1;
        private CustomExternalTextureRenderer renderer = null;
        private long timestampOffset = 0L;
        private long previousTimestamp = 0L;
        private boolean previousTimestampValid = false;
        protected int destinationWidth = 0;
        protected int destinationHeight = 0;

        public RenderThread(EGLContext parentContext, int numBuffers,int rotation) {
            super(parentContext);
            this.outputFrames = new ArrayList();
            this.outputFrames.addAll(Collections.nCopies(numBuffers, null));
            this.renderer = new CustomExternalTextureRenderer();
            renderer.setRotation(rotation);
            this.consumers = new ArrayList();
        }

        public void setFlipY(boolean flip) {
            this.renderer.setFlipY(flip);
        }

        public void setSurfaceTexture(SurfaceTexture texture, int width, int height) {
            if (this.surfaceTexture != null) {
                this.surfaceTexture.setOnFrameAvailableListener((SurfaceTexture.OnFrameAvailableListener)null);
            }

            this.surfaceTexture = texture;
            if (this.surfaceTexture != null) {
                this.surfaceTexture.setOnFrameAvailableListener(this);
            }

            this.destinationWidth = width;
            this.destinationHeight = height;
        }

        public void setSurfaceTextureAndAttachToGLContext(SurfaceTexture texture, int width, int height) {
            this.setSurfaceTexture(texture, width, height);
            int[] textures = new int[1];
            GLES20.glGenTextures(1, textures, 0);
            this.surfaceTexture.attachToGLContext(textures[0]);
        }

        public void setConsumer(TextureFrameConsumer consumer) {
            synchronized(this.consumers) {
                this.consumers.clear();
                this.consumers.add(consumer);
            }
        }

        public void addConsumer(TextureFrameConsumer consumer) {
            synchronized(this.consumers) {
                this.consumers.add(consumer);
            }
        }

        public void removeConsumer(TextureFrameConsumer consumer) {
            synchronized(this.consumers) {
                this.consumers.remove(consumer);
            }
        }

        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
            this.handler.post(() -> {
                this.renderNext(surfaceTexture);
            });
        }

        public void prepareGl() {
            super.prepareGl();
            GLES20.glClearColor(0.0F, 0.0F, 0.0F, 1.0F);
            this.renderer.setup();
        }

        public void releaseGl() {
            for(int i = 0; i < this.outputFrames.size(); ++i) {
                this.teardownDestination(i);
            }

            this.renderer.release();
            super.releaseGl();
        }

        @SuppressLint("WrongConstant")
        protected void renderNext(SurfaceTexture fromTexture) {
            if (fromTexture == this.surfaceTexture) {
                synchronized(this.consumers) {
                    boolean frameUpdated = false;
                    Iterator var4 = this.consumers.iterator();

                    while(var4.hasNext()) {
                        TextureFrameConsumer consumer = (TextureFrameConsumer)var4.next();
                        AppTextureFrame outputFrame = this.nextOutputFrame();
                        this.updateOutputFrame(outputFrame);
                        frameUpdated = true;
                        if (consumer != null) {
                            if (Log.isLoggable("ExternalTextureConv", 2)) {
                                Log.v("ExternalTextureConv", String.format("Locking tex: %d width: %d height: %d", outputFrame.getTextureName(), outputFrame.getWidth(), outputFrame.getHeight()));
                            }

                            outputFrame.setInUse();
                            consumer.onNewFrame(outputFrame);
                        }
                    }

                    if (!frameUpdated) {
                        AppTextureFrame outputFrame = this.nextOutputFrame();
                        this.updateOutputFrame(outputFrame);
                    }

                }
            }
        }

        private void teardownDestination(int index) {
            if (this.outputFrames.get(index) != null) {
                this.waitUntilReleased((AppTextureFrame)this.outputFrames.get(index));
                GLES20.glDeleteTextures(1, new int[]{((AppTextureFrame)this.outputFrames.get(index)).getTextureName()}, 0);
                this.outputFrames.set(index, null);
            }

        }

        private void setupDestination(int index) {
            this.teardownDestination(index);
            int destinationTextureId = ShaderUtil.createRgbaTexture(this.destinationWidth, this.destinationHeight);
            Log.d("ExternalTextureConv", String.format("Created output texture: %d width: %d height: %d", destinationTextureId, this.destinationWidth, this.destinationHeight));
            this.bindFramebuffer(destinationTextureId, this.destinationWidth, this.destinationHeight);
            this.outputFrames.set(index, new AppTextureFrame(destinationTextureId, this.destinationWidth, this.destinationHeight));
        }

        private AppTextureFrame nextOutputFrame() {
            this.outputFrameIndex = (this.outputFrameIndex + 1) % this.outputFrames.size();
            AppTextureFrame outputFrame = (AppTextureFrame)this.outputFrames.get(this.outputFrameIndex);
            if (outputFrame == null || outputFrame.getWidth() != this.destinationWidth || outputFrame.getHeight() != this.destinationHeight) {
                this.setupDestination(this.outputFrameIndex);
                outputFrame = (AppTextureFrame)this.outputFrames.get(this.outputFrameIndex);
            }

            this.waitUntilReleased(outputFrame);
            return outputFrame;
        }

        private void updateOutputFrame(AppTextureFrame outputFrame) {
            this.bindFramebuffer(outputFrame.getTextureName(), this.destinationWidth, this.destinationHeight);
            this.renderer.render(this.surfaceTexture);
            long textureTimestamp = this.surfaceTexture.getTimestamp() / 1000L;
            if (this.previousTimestampValid && textureTimestamp + this.timestampOffset <= this.previousTimestamp) {
                this.timestampOffset = this.previousTimestamp + 1L - textureTimestamp;
            }

            outputFrame.setTimestamp(textureTimestamp + this.timestampOffset);
            this.previousTimestamp = outputFrame.getTimestamp();
            this.previousTimestampValid = true;
        }

        @SuppressLint("WrongConstant")
        private void waitUntilReleased(AppTextureFrame frame) {
            try {
                if (Log.isLoggable("ExternalTextureConv", 2)) {
                    Log.v("ExternalTextureConv", String.format("Waiting for tex: %d width: %d height: %d", frame.getTextureName(), frame.getWidth(), frame.getHeight()));
                }

                frame.waitUntilReleased();
                if (Log.isLoggable("ExternalTextureConv", 2)) {
                    Log.v("ExternalTextureConv", String.format("Finished waiting for tex: %d width: %d height: %d", frame.getTextureName(), frame.getWidth(), frame.getHeight()));
                }

            } catch (InterruptedException var3) {
                Thread.currentThread().interrupt();
                Log.e("ExternalTextureConv", "thread was unexpectedly interrupted: " + var3.getMessage());
                throw new RuntimeException(var3);
            }
        }
    }
}

在MainActivity里 修改onResume() 将converter修改成CustomExternalTextureConverter 这样就是摄像头和预览画面就横屏了

override fun onResume() {
    super.onResume()
    //改动部分
    converter = CustomExternalTextureConverter(eglManager!!.context,2,270)

    converter!!.setFlipY(FLIP_FRAMES_VERTICALLY)
    converter!!.setConsumer(processor)

    
    if (PermissionHelper.cameraPermissionsGranted(this)) {
        startCamera()
    }
}